[llvm] [AMDGPU] Handle amdgpu.last.use metadata (PR #83816)
Mirko BrkuĊĦanin via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 4 02:10:23 PST 2024
https://github.com/mbrkusanin created https://github.com/llvm/llvm-project/pull/83816
Convert !amdgpu.last.use metadata into Machine Memory Operand for last use
and handle it in SIMemoryLegalizer similar to nontemporal and volatile.
>From f9540d55f5b834c6ae782524de3132a0baa3735c Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Fri, 1 Mar 2024 17:58:53 +0100
Subject: [PATCH 1/4] [AMDGPU] Add GFX12 memory legalizer tests
---
.../CodeGen/AMDGPU/memory-legalizer-fence.ll | 352 +++
.../AMDGPU/memory-legalizer-flat-agent.ll | 2102 ++++++++++++++++
.../memory-legalizer-flat-nontemporal.ll | 102 +
.../memory-legalizer-flat-singlethread.ll | 1866 ++++++++++++++
.../AMDGPU/memory-legalizer-flat-system.ll | 2102 ++++++++++++++++
.../AMDGPU/memory-legalizer-flat-volatile.ll | 156 ++
.../AMDGPU/memory-legalizer-flat-wavefront.ll | 1844 ++++++++++++++
.../AMDGPU/memory-legalizer-flat-workgroup.ll | 1918 ++++++++++++++
.../AMDGPU/memory-legalizer-global-agent.ll | 2216 +++++++++++++++++
.../memory-legalizer-global-nontemporal.ll | 98 +
.../memory-legalizer-global-singlethread.ll | 2170 ++++++++++++++++
.../AMDGPU/memory-legalizer-global-system.ll | 2128 ++++++++++++++++
.../memory-legalizer-global-volatile.ll | 151 ++
.../memory-legalizer-global-wavefront.ll | 2170 ++++++++++++++++
.../memory-legalizer-global-workgroup.ll | 2206 ++++++++++++++++
.../AMDGPU/memory-legalizer-local-agent.ll | 1836 ++++++++++++++
.../memory-legalizer-local-nontemporal.ll | 102 +
.../memory-legalizer-local-singlethread.ll | 1770 +++++++++++++
.../AMDGPU/memory-legalizer-local-system.ll | 1836 ++++++++++++++
.../AMDGPU/memory-legalizer-local-volatile.ll | 139 ++
.../memory-legalizer-local-wavefront.ll | 1770 +++++++++++++
.../memory-legalizer-local-workgroup.ll | 1836 ++++++++++++++
22 files changed, 30870 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
index 77962fadcacfc6..e13542f61474e2 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @singlethread_acquire_fence() {
; GFX6-LABEL: singlethread_acquire_fence:
@@ -55,6 +57,14 @@ define amdgpu_kernel void @singlethread_acquire_fence() {
; GFX11-CU-LABEL: singlethread_acquire_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: singlethread_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: singlethread_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("singlethread") acquire
ret void
@@ -104,6 +114,14 @@ define amdgpu_kernel void @singlethread_release_fence() {
; GFX11-CU-LABEL: singlethread_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: singlethread_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: singlethread_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("singlethread") release
ret void
@@ -153,6 +171,14 @@ define amdgpu_kernel void @singlethread_acq_rel_fence() {
; GFX11-CU-LABEL: singlethread_acq_rel_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: singlethread_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: singlethread_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("singlethread") acq_rel
ret void
@@ -202,6 +228,14 @@ define amdgpu_kernel void @singlethread_seq_cst_fence() {
; GFX11-CU-LABEL: singlethread_seq_cst_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: singlethread_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: singlethread_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("singlethread") seq_cst
ret void
@@ -251,6 +285,14 @@ define amdgpu_kernel void @singlethread_one_as_acquire_fence() {
; GFX11-CU-LABEL: singlethread_one_as_acquire_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: singlethread_one_as_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: singlethread_one_as_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("singlethread-one-as") acquire
ret void
@@ -300,6 +342,14 @@ define amdgpu_kernel void @singlethread_one_as_release_fence() {
; GFX11-CU-LABEL: singlethread_one_as_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: singlethread_one_as_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: singlethread_one_as_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("singlethread-one-as") release
ret void
@@ -349,6 +399,14 @@ define amdgpu_kernel void @singlethread_one_as_acq_rel_fence() {
; GFX11-CU-LABEL: singlethread_one_as_acq_rel_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: singlethread_one_as_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: singlethread_one_as_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("singlethread-one-as") acq_rel
ret void
@@ -398,6 +456,14 @@ define amdgpu_kernel void @singlethread_one_as_seq_cst_fence() {
; GFX11-CU-LABEL: singlethread_one_as_seq_cst_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: singlethread_one_as_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: singlethread_one_as_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("singlethread-one-as") seq_cst
ret void
@@ -447,6 +513,14 @@ define amdgpu_kernel void @wavefront_acquire_fence() {
; GFX11-CU-LABEL: wavefront_acquire_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: wavefront_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: wavefront_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("wavefront") acquire
ret void
@@ -496,6 +570,14 @@ define amdgpu_kernel void @wavefront_release_fence() {
; GFX11-CU-LABEL: wavefront_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: wavefront_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: wavefront_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("wavefront") release
ret void
@@ -545,6 +627,14 @@ define amdgpu_kernel void @wavefront_acq_rel_fence() {
; GFX11-CU-LABEL: wavefront_acq_rel_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: wavefront_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: wavefront_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("wavefront") acq_rel
ret void
@@ -594,6 +684,14 @@ define amdgpu_kernel void @wavefront_seq_cst_fence() {
; GFX11-CU-LABEL: wavefront_seq_cst_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: wavefront_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: wavefront_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("wavefront") seq_cst
ret void
@@ -643,6 +741,14 @@ define amdgpu_kernel void @wavefront_one_as_acquire_fence() {
; GFX11-CU-LABEL: wavefront_one_as_acquire_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: wavefront_one_as_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: wavefront_one_as_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("wavefront-one-as") acquire
ret void
@@ -692,6 +798,14 @@ define amdgpu_kernel void @wavefront_one_as_release_fence() {
; GFX11-CU-LABEL: wavefront_one_as_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: wavefront_one_as_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: wavefront_one_as_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("wavefront-one-as") release
ret void
@@ -741,6 +855,14 @@ define amdgpu_kernel void @wavefront_one_as_acq_rel_fence() {
; GFX11-CU-LABEL: wavefront_one_as_acq_rel_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: wavefront_one_as_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: wavefront_one_as_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("wavefront-one-as") acq_rel
ret void
@@ -790,6 +912,14 @@ define amdgpu_kernel void @wavefront_one_as_seq_cst_fence() {
; GFX11-CU-LABEL: wavefront_one_as_seq_cst_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: wavefront_one_as_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: wavefront_one_as_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("wavefront-one-as") seq_cst
ret void
@@ -843,6 +973,15 @@ define amdgpu_kernel void @workgroup_acquire_fence() {
; GFX11-CU-LABEL: workgroup_acquire_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: workgroup_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: workgroup_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("workgroup") acquire
ret void
@@ -892,6 +1031,14 @@ define amdgpu_kernel void @workgroup_release_fence() {
; GFX11-CU-LABEL: workgroup_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: workgroup_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: workgroup_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("workgroup") release
ret void
@@ -945,6 +1092,15 @@ define amdgpu_kernel void @workgroup_acq_rel_fence() {
; GFX11-CU-LABEL: workgroup_acq_rel_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: workgroup_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: workgroup_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("workgroup") acq_rel
ret void
@@ -998,6 +1154,15 @@ define amdgpu_kernel void @workgroup_seq_cst_fence() {
; GFX11-CU-LABEL: workgroup_seq_cst_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: workgroup_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: workgroup_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("workgroup") seq_cst
ret void
@@ -1051,6 +1216,15 @@ define amdgpu_kernel void @workgroup_one_as_acquire_fence() {
; GFX11-CU-LABEL: workgroup_one_as_acquire_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: workgroup_one_as_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: workgroup_one_as_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("workgroup-one-as") acquire
ret void
@@ -1100,6 +1274,14 @@ define amdgpu_kernel void @workgroup_one_as_release_fence() {
; GFX11-CU-LABEL: workgroup_one_as_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: workgroup_one_as_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: workgroup_one_as_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("workgroup-one-as") release
ret void
@@ -1153,6 +1335,15 @@ define amdgpu_kernel void @workgroup_one_as_acq_rel_fence() {
; GFX11-CU-LABEL: workgroup_one_as_acq_rel_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("workgroup-one-as") acq_rel
ret void
@@ -1206,6 +1397,15 @@ define amdgpu_kernel void @workgroup_one_as_seq_cst_fence() {
; GFX11-CU-LABEL: workgroup_one_as_seq_cst_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("workgroup-one-as") seq_cst
ret void
@@ -1269,6 +1469,16 @@ define amdgpu_kernel void @agent_acquire_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: agent_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: agent_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("agent") acquire
ret void
@@ -1320,6 +1530,14 @@ define amdgpu_kernel void @agent_release_fence() {
; GFX11-CU-LABEL: agent_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: agent_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: agent_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("agent") release
ret void
@@ -1385,6 +1603,16 @@ define amdgpu_kernel void @agent_acq_rel_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: agent_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: agent_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("agent") acq_rel
ret void
@@ -1450,6 +1678,16 @@ define amdgpu_kernel void @agent_seq_cst_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: agent_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: agent_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("agent") seq_cst
ret void
@@ -1513,6 +1751,16 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: agent_one_as_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: agent_one_as_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("agent-one-as") acquire
ret void
@@ -1564,6 +1812,14 @@ define amdgpu_kernel void @agent_one_as_release_fence() {
; GFX11-CU-LABEL: agent_one_as_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: agent_one_as_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: agent_one_as_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("agent-one-as") release
ret void
@@ -1629,6 +1885,16 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: agent_one_as_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: agent_one_as_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("agent-one-as") acq_rel
ret void
@@ -1694,6 +1960,16 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: agent_one_as_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: agent_one_as_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("agent-one-as") seq_cst
ret void
@@ -1759,6 +2035,16 @@ define amdgpu_kernel void @system_acquire_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: system_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: system_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
entry:
fence acquire
ret void
@@ -1812,6 +2098,14 @@ define amdgpu_kernel void @system_release_fence() {
; GFX11-CU-LABEL: system_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: system_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: system_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence release
ret void
@@ -1881,6 +2175,16 @@ define amdgpu_kernel void @system_acq_rel_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: system_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: system_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
entry:
fence acq_rel
ret void
@@ -1950,6 +2254,16 @@ define amdgpu_kernel void @system_seq_cst_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: system_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: system_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
entry:
fence seq_cst
ret void
@@ -2015,6 +2329,16 @@ define amdgpu_kernel void @system_one_as_acquire_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: system_one_as_acquire_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: system_one_as_acquire_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("one-as") acquire
ret void
@@ -2068,6 +2392,14 @@ define amdgpu_kernel void @system_one_as_release_fence() {
; GFX11-CU-LABEL: system_one_as_release_fence:
; GFX11-CU: ; %bb.0: ; %entry
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: system_one_as_release_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: system_one_as_release_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("one-as") release
ret void
@@ -2137,6 +2469,16 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: system_one_as_acq_rel_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: system_one_as_acq_rel_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("one-as") acq_rel
ret void
@@ -2206,6 +2548,16 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() {
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: system_one_as_seq_cst_fence:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: system_one_as_seq_cst_fence:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
entry:
fence syncscope("one-as") seq_cst
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
index d9f5e64eae9278..a922c8cd2f30a5 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
@@ -9,6 +9,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @flat_agent_unordered_load(
; GFX7-LABEL: flat_agent_unordered_load:
@@ -136,6 +138,28 @@ define amdgpu_kernel void @flat_agent_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("agent") unordered, align 4
@@ -269,6 +293,28 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("agent") monotonic, align 4
@@ -415,6 +461,30 @@ define amdgpu_kernel void @flat_agent_acquire_load(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("agent") acquire, align 4
@@ -561,6 +631,30 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("agent") seq_cst, align 4
@@ -676,6 +770,28 @@ define amdgpu_kernel void @flat_agent_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("agent") unordered, align 4
@@ -790,6 +906,28 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("agent") monotonic, align 4
@@ -906,6 +1044,28 @@ define amdgpu_kernel void @flat_agent_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("agent") release, align 4
@@ -1022,6 +1182,28 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("agent") seq_cst, align 4
@@ -1136,6 +1318,24 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") monotonic
@@ -1277,6 +1477,28 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acquire
@@ -1393,6 +1615,24 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") release
@@ -1536,6 +1776,28 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acq_rel
@@ -1679,6 +1941,28 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst
@@ -1826,6 +2110,30 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acquire
@@ -1976,6 +2284,30 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acq_rel
@@ -2126,6 +2458,30 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst
@@ -2239,6 +2595,24 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2379,6 +2753,28 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2494,6 +2890,24 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2636,6 +3050,28 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2778,6 +3214,28 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2918,6 +3376,28 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3058,6 +3538,28 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3200,6 +3702,28 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3342,6 +3866,28 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3484,6 +4030,28 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3626,6 +4194,28 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3768,6 +4358,28 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3910,6 +4522,28 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4052,6 +4686,28 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4194,6 +4850,28 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4335,6 +5013,28 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4491,6 +5191,30 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4636,6 +5360,28 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4794,6 +5540,30 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4952,6 +5722,30 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5108,6 +5902,30 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5264,6 +6082,30 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5422,6 +6264,30 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5580,6 +6446,30 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5738,6 +6628,30 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5896,6 +6810,30 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6054,6 +6992,30 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6212,6 +7174,30 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6370,6 +7356,30 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6528,6 +7538,30 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6663,6 +7697,28 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("agent-one-as") unordered, align 4
@@ -6796,6 +7852,28 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("agent-one-as") monotonic, align 4
@@ -6950,6 +8028,32 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("agent-one-as") acquire, align 4
@@ -7104,6 +8208,32 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("agent-one-as") seq_cst, align 4
@@ -7219,6 +8349,28 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("agent-one-as") unordered, align 4
@@ -7333,6 +8485,28 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("agent-one-as") monotonic, align 4
@@ -7449,6 +8623,28 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("agent-one-as") release, align 4
@@ -7565,6 +8761,28 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("agent-one-as") seq_cst, align 4
@@ -7679,6 +8897,24 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") monotonic
@@ -7816,6 +9052,28 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acquire
@@ -7932,6 +9190,24 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") release
@@ -8071,6 +9347,28 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -8210,6 +9508,28 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -8364,6 +9684,32 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acquire
@@ -8521,6 +9867,32 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -8678,6 +10050,32 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -8791,6 +10189,24 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8927,6 +10343,28 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9042,6 +10480,24 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9180,6 +10636,28 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9318,6 +10796,28 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9454,6 +10954,28 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9590,6 +11112,28 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9728,6 +11272,28 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9866,6 +11432,28 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10004,6 +11592,28 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10142,6 +11752,28 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10280,6 +11912,28 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10418,6 +12072,28 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10556,6 +12232,28 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10694,6 +12392,28 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10835,6 +12555,28 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10999,6 +12741,32 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11144,6 +12912,28 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11310,6 +13100,32 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11476,6 +13292,32 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11640,6 +13482,32 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11804,6 +13672,32 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11970,6 +13864,32 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12136,6 +14056,32 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12302,6 +14248,32 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12468,6 +14440,32 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12634,6 +14632,32 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12800,6 +14824,32 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12966,6 +15016,32 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -13132,6 +15208,32 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index a59c0394bebe20..9f0556023aa24a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -9,6 +9,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @flat_nontemporal_load_0(
; GFX7-LABEL: flat_nontemporal_load_0:
@@ -136,6 +138,28 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_load_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_load_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load i32, ptr %in, align 4, !nontemporal !0
@@ -287,6 +311,34 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_load_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-WGP-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_load_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-CU-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -422,6 +474,28 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 glc slc dlc
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_store_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 th:TH_STORE_NT
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_store_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 th:TH_STORE_NT
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load i32, ptr %in, align 4
@@ -573,6 +647,34 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 glc slc dlc
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_store_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-WGP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-WGP-NEXT: v_add_co_u32 v0, s0, s2, v0
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[1:2]
+; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 th:TH_STORE_NT
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_store_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-CU-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-CU-NEXT: v_add_co_u32 v0, s0, s2, v0
+; GFX12-CU-NEXT: flat_load_b32 v2, v[1:2]
+; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 th:TH_STORE_NT
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
index c4c01968d5c398..84438d58d7e7dd 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
@@ -9,6 +9,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @flat_singlethread_unordered_load(
; GFX7-LABEL: flat_singlethread_unordered_load:
@@ -136,6 +138,28 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("singlethread") unordered, align 4
@@ -269,6 +293,28 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("singlethread") monotonic, align 4
@@ -402,6 +448,28 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("singlethread") acquire, align 4
@@ -535,6 +603,28 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("singlethread") seq_cst, align 4
@@ -650,6 +740,28 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("singlethread") unordered, align 4
@@ -764,6 +876,28 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("singlethread") monotonic, align 4
@@ -878,6 +1012,28 @@ define amdgpu_kernel void @flat_singlethread_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("singlethread") release, align 4
@@ -992,6 +1148,28 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("singlethread") seq_cst, align 4
@@ -1106,6 +1284,24 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") monotonic
@@ -1220,6 +1416,24 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") acquire
@@ -1334,6 +1548,24 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") release
@@ -1448,6 +1680,24 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") acq_rel
@@ -1562,6 +1812,24 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") seq_cst
@@ -1696,6 +1964,28 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") acquire
@@ -1831,6 +2121,28 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") acq_rel
@@ -1966,6 +2278,28 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") seq_cst
@@ -2079,6 +2413,24 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2192,6 +2544,24 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2305,6 +2675,24 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2418,6 +2806,24 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2531,6 +2937,24 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2644,6 +3068,24 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2757,6 +3199,24 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2870,6 +3330,24 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2983,6 +3461,24 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3096,6 +3592,24 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3209,6 +3723,24 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3322,6 +3854,24 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3435,6 +3985,24 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3548,6 +4116,24 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3661,6 +4247,24 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3802,6 +4406,28 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3945,6 +4571,28 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4088,6 +4736,28 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4231,6 +4901,28 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4374,6 +5066,28 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4517,6 +5231,28 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4660,6 +5396,28 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4803,6 +5561,28 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4946,6 +5726,28 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5089,6 +5891,28 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5232,6 +6056,28 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5375,6 +6221,28 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5518,6 +6386,28 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5661,6 +6551,28 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5804,6 +6716,28 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5939,6 +6873,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("singlethread-one-as") unordered, align 4
@@ -6072,6 +7028,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("singlethread-one-as") monotonic, align 4
@@ -6205,6 +7183,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("singlethread-one-as") acquire, align 4
@@ -6338,6 +7338,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("singlethread-one-as") seq_cst, align 4
@@ -6453,6 +7475,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("singlethread-one-as") unordered, align 4
@@ -6567,6 +7611,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("singlethread-one-as") monotonic, align 4
@@ -6681,6 +7747,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("singlethread-one-as") release, align 4
@@ -6795,6 +7883,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("singlethread-one-as") seq_cst, align 4
@@ -6909,6 +8019,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") monotonic
@@ -7023,6 +8151,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -7137,6 +8283,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") release
@@ -7251,6 +8415,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -7365,6 +8547,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -7499,6 +8699,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -7634,6 +8856,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -7769,6 +9013,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -7882,6 +9148,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -7995,6 +9279,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8108,6 +9410,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8221,6 +9541,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8334,6 +9672,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8447,6 +9803,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8560,6 +9934,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8673,6 +10065,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8786,6 +10196,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8899,6 +10327,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9012,6 +10458,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9125,6 +10589,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9238,6 +10720,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9351,6 +10851,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9464,6 +10982,24 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9605,6 +11141,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9748,6 +11306,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9891,6 +11471,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10034,6 +11636,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10177,6 +11801,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10320,6 +11966,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10463,6 +12131,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10606,6 +12296,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10749,6 +12461,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10892,6 +12626,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11035,6 +12791,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11178,6 +12956,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11321,6 +13121,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11464,6 +13286,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11607,6 +13451,28 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
index 372f9ada5839a3..57a2e72c907e14 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
@@ -9,6 +9,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @flat_system_unordered_load(
; GFX7-LABEL: flat_system_unordered_load:
@@ -136,6 +138,28 @@ define amdgpu_kernel void @flat_system_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in unordered, align 4
@@ -269,6 +293,28 @@ define amdgpu_kernel void @flat_system_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in monotonic, align 4
@@ -417,6 +463,30 @@ define amdgpu_kernel void @flat_system_acquire_load(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in acquire, align 4
@@ -565,6 +635,30 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in seq_cst, align 4
@@ -680,6 +774,28 @@ define amdgpu_kernel void @flat_system_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out unordered, align 4
@@ -794,6 +910,28 @@ define amdgpu_kernel void @flat_system_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out monotonic, align 4
@@ -912,6 +1050,28 @@ define amdgpu_kernel void @flat_system_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out release, align 4
@@ -1030,6 +1190,28 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out seq_cst, align 4
@@ -1144,6 +1326,24 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in monotonic
@@ -1287,6 +1487,28 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in acquire
@@ -1405,6 +1627,24 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in release
@@ -1552,6 +1792,28 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in acq_rel
@@ -1699,6 +1961,28 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in seq_cst
@@ -1848,6 +2132,30 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in acquire
@@ -2002,6 +2310,30 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in acq_rel
@@ -2156,6 +2488,30 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in seq_cst
@@ -2269,6 +2625,24 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2411,6 +2785,28 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2528,6 +2924,24 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2674,6 +3088,28 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2820,6 +3256,28 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2962,6 +3420,28 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3104,6 +3584,28 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3250,6 +3752,28 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3396,6 +3920,28 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3542,6 +4088,28 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3688,6 +4256,28 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3834,6 +4424,28 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3980,6 +4592,28 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4126,6 +4760,28 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4272,6 +4928,28 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4413,6 +5091,28 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4571,6 +5271,30 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4718,6 +5442,28 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4880,6 +5626,30 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5042,6 +5812,30 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5200,6 +5994,30 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5358,6 +6176,30 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5520,6 +6362,30 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5682,6 +6548,30 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5844,6 +6734,30 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6006,6 +6920,30 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6168,6 +7106,30 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6330,6 +7292,30 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6492,6 +7478,30 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6654,6 +7664,30 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -6789,6 +7823,28 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("one-as") unordered, align 4
@@ -6922,6 +7978,28 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("one-as") monotonic, align 4
@@ -7078,6 +8156,32 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("one-as") acquire, align 4
@@ -7234,6 +8338,32 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("one-as") seq_cst, align 4
@@ -7349,6 +8479,28 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("one-as") unordered, align 4
@@ -7463,6 +8615,28 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("one-as") monotonic, align 4
@@ -7581,6 +8755,28 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("one-as") release, align 4
@@ -7699,6 +8895,28 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("one-as") seq_cst, align 4
@@ -7813,6 +9031,24 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") monotonic
@@ -7952,6 +9188,28 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") acquire
@@ -8070,6 +9328,24 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") release
@@ -8213,6 +9489,28 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") acq_rel
@@ -8356,6 +9654,28 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") seq_cst
@@ -8512,6 +9832,32 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") acquire
@@ -8673,6 +10019,32 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") acq_rel
@@ -8834,6 +10206,32 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") seq_cst
@@ -8947,6 +10345,24 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9085,6 +10501,28 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9202,6 +10640,24 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9344,6 +10800,28 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9486,6 +10964,28 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9624,6 +11124,28 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9762,6 +11284,28 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9904,6 +11448,28 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10046,6 +11612,28 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10188,6 +11776,28 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10330,6 +11940,28 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10472,6 +12104,28 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10614,6 +12268,28 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10756,6 +12432,28 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10898,6 +12596,28 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11039,6 +12759,28 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11205,6 +12947,32 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11352,6 +13120,28 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11522,6 +13312,32 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11692,6 +13508,32 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11858,6 +13700,32 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12024,6 +13892,32 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12194,6 +14088,32 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12364,6 +14284,32 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12534,6 +14480,32 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12704,6 +14676,32 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -12874,6 +14872,32 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -13044,6 +15068,32 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -13214,6 +15264,32 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -13384,6 +15460,32 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
index 3c8b3aab6ad02f..735e541b9f7c95 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
@@ -5,6 +5,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @flat_nontemporal_load_0(
; GFX7-LABEL: flat_nontemporal_load_0:
@@ -86,6 +88,30 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_load_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_load_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load volatile i32, ptr %in, align 4
@@ -185,6 +211,36 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_load_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-WGP-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_load_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-CU-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -274,6 +330,30 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 dlc
; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_store_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_store_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load i32, ptr %in, align 4
@@ -373,6 +453,36 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 dlc
; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_store_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-WGP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-WGP-NEXT: v_add_co_u32 v0, s0, s2, v0
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[1:2]
+; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_store_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-CU-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-CU-NEXT: v_add_co_u32 v0, s0, s2, v0
+; GFX12-CU-NEXT: flat_load_b32 v2, v[1:2]
+; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -462,6 +572,30 @@ define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_volatile_workgroup_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_volatile_workgroup_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic volatile i32, ptr %in syncscope("workgroup") acquire, align 4
@@ -537,6 +671,28 @@ define amdgpu_kernel void @flat_volatile_workgroup_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_volatile_workgroup_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_volatile_workgroup_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic volatile i32 %in, ptr %out syncscope("workgroup") release, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
index 0881f3665f3ad1..9a1209bc47e88a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
@@ -9,6 +9,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @flat_wavefront_unordered_load(
; GFX7-LABEL: flat_wavefront_unordered_load:
@@ -136,6 +138,28 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("wavefront") unordered, align 4
@@ -269,6 +293,28 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("wavefront") monotonic, align 4
@@ -402,6 +448,28 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("wavefront") acquire, align 4
@@ -535,6 +603,28 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("wavefront") seq_cst, align 4
@@ -650,6 +740,28 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("wavefront") unordered, align 4
@@ -764,6 +876,28 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("wavefront") monotonic, align 4
@@ -878,6 +1012,28 @@ define amdgpu_kernel void @flat_wavefront_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("wavefront") release, align 4
@@ -992,6 +1148,28 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("wavefront") seq_cst, align 4
@@ -1106,6 +1284,24 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") monotonic
@@ -1220,6 +1416,24 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acquire
@@ -1334,6 +1548,24 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") release
@@ -1448,6 +1680,24 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acq_rel
@@ -1562,6 +1812,24 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") seq_cst
@@ -1696,6 +1964,28 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acquire
@@ -1831,6 +2121,28 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acq_rel
@@ -1966,6 +2278,28 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") seq_cst
@@ -2079,6 +2413,24 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2192,6 +2544,24 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2305,6 +2675,24 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2418,6 +2806,24 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2531,6 +2937,24 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2644,6 +3068,24 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2757,6 +3199,24 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2870,6 +3330,24 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2983,6 +3461,24 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3096,6 +3592,24 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3209,6 +3723,24 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3322,6 +3854,24 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3435,6 +3985,24 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3548,6 +4116,24 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3661,6 +4247,24 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3802,6 +4406,28 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3945,6 +4571,28 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4088,6 +4736,28 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4231,6 +4901,28 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4374,6 +5066,28 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4517,6 +5231,28 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4660,6 +5396,28 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4803,6 +5561,28 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4946,6 +5726,28 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5089,6 +5891,28 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5232,6 +6056,28 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5375,6 +6221,28 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5518,6 +6386,28 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5661,6 +6551,28 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5804,6 +6716,28 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5939,6 +6873,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("wavefront-one-as") unordered, align 4
@@ -6072,6 +7028,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("wavefront-one-as") monotonic, align 4
@@ -6205,6 +7183,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("wavefront-one-as") acquire, align 4
@@ -6338,6 +7338,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("wavefront-one-as") seq_cst, align 4
@@ -6453,6 +7475,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("wavefront-one-as") unordered, align 4
@@ -6567,6 +7611,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("wavefront-one-as") monotonic, align 4
@@ -6681,6 +7747,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("wavefront-one-as") release, align 4
@@ -6795,6 +7883,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("wavefront-one-as") seq_cst, align 4
@@ -6909,6 +8019,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") monotonic
@@ -7023,6 +8151,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -7137,6 +8283,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") release
@@ -7251,6 +8415,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -7365,6 +8547,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -7499,6 +8699,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -7634,6 +8856,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -7769,6 +9013,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -7882,6 +9148,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -7995,6 +9279,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8108,6 +9410,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8221,6 +9541,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8334,6 +9672,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8447,6 +9803,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8560,6 +9934,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8673,6 +10065,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8786,6 +10196,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8899,6 +10327,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9012,6 +10458,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9125,6 +10589,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9238,6 +10720,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9351,6 +10851,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9464,6 +10982,24 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9605,6 +11141,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9748,6 +11306,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9891,6 +11471,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10034,6 +11636,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10177,6 +11801,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10320,6 +11966,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10463,6 +12131,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10606,6 +12296,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10749,6 +12461,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10892,6 +12626,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11035,6 +12791,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11178,6 +12956,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11321,6 +13121,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11464,6 +13286,28 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
index 420819b12d1254..13affefad5dfc5 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
@@ -9,6 +9,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @flat_workgroup_unordered_load(
; GFX7-LABEL: flat_workgroup_unordered_load:
@@ -136,6 +138,28 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("workgroup") unordered, align 4
@@ -269,6 +293,28 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("workgroup") monotonic, align 4
@@ -412,6 +458,30 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("workgroup") acquire, align 4
@@ -555,6 +625,30 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("workgroup") seq_cst, align 4
@@ -670,6 +764,28 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("workgroup") unordered, align 4
@@ -784,6 +900,28 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("workgroup") monotonic, align 4
@@ -898,6 +1036,28 @@ define amdgpu_kernel void @flat_workgroup_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("workgroup") release, align 4
@@ -1012,6 +1172,28 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("workgroup") seq_cst, align 4
@@ -1126,6 +1308,24 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") monotonic
@@ -1256,6 +1456,27 @@ define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acquire
@@ -1370,6 +1591,24 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") release
@@ -1500,6 +1739,27 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acq_rel
@@ -1630,6 +1890,27 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") seq_cst
@@ -1768,6 +2049,29 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acquire
@@ -1907,6 +2211,29 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acq_rel
@@ -2046,6 +2373,29 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") seq_cst
@@ -2159,6 +2509,24 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2288,6 +2656,27 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2401,6 +2790,24 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2530,6 +2937,27 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2659,6 +3087,27 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2788,6 +3237,27 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -2917,6 +3387,27 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3046,6 +3537,27 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3175,6 +3687,27 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3304,6 +3837,27 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3433,6 +3987,27 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3574,6 +4149,28 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3724,6 +4321,29 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -3867,6 +4487,28 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4017,6 +4659,29 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4167,6 +4832,29 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4317,6 +5005,29 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4467,6 +5178,29 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4617,6 +5351,29 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4767,6 +5524,29 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -4917,6 +5697,29 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5067,6 +5870,29 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5217,6 +6043,29 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5367,6 +6216,29 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5517,6 +6389,29 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5667,6 +6562,29 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -5802,6 +6720,28 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("workgroup-one-as") unordered, align 4
@@ -5935,6 +6875,28 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("workgroup-one-as") monotonic, align 4
@@ -6074,6 +7036,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("workgroup-one-as") acquire, align 4
@@ -6213,6 +7199,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %in, ptr %out) {
entry:
%val = load atomic i32, ptr %in syncscope("workgroup-one-as") seq_cst, align 4
@@ -6328,6 +7338,28 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("workgroup-one-as") unordered, align 4
@@ -6442,6 +7474,28 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("workgroup-one-as") monotonic, align 4
@@ -6556,6 +7610,28 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("workgroup-one-as") release, align 4
@@ -6670,6 +7746,28 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr %out) {
entry:
store atomic i32 %in, ptr %out syncscope("workgroup-one-as") seq_cst, align 4
@@ -6784,6 +7882,24 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") monotonic
@@ -6906,6 +8022,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -7020,6 +8156,24 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") release
@@ -7142,6 +8296,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -7264,6 +8438,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -7404,6 +8598,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -7545,6 +8763,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -7686,6 +8928,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -7799,6 +9065,24 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -7920,6 +9204,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8033,6 +9337,24 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8154,6 +9476,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8275,6 +9617,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8396,6 +9758,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8517,6 +9899,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8638,6 +10040,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8759,6 +10181,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -8880,6 +10322,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9001,6 +10463,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9122,6 +10604,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9243,6 +10745,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9364,6 +10886,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9485,6 +11027,26 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9626,6 +11188,28 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9775,6 +11359,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -9918,6 +11526,28 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10067,6 +11697,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10216,6 +11870,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10365,6 +12043,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10514,6 +12216,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10663,6 +12389,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10812,6 +12562,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -10961,6 +12735,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11110,6 +12908,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11259,6 +13081,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11408,6 +13254,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11557,6 +13427,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
@@ -11706,6 +13600,30 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
ptr %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
index 0449e41682967d..d02b937dcc928a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @global_agent_unordered_load(
; GFX6-LABEL: global_agent_unordered_load:
@@ -138,6 +140,30 @@ define amdgpu_kernel void @global_agent_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("agent") unordered, align 4
@@ -272,6 +298,30 @@ define amdgpu_kernel void @global_agent_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("agent") monotonic, align 4
@@ -420,6 +470,32 @@ define amdgpu_kernel void @global_agent_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("agent") acquire, align 4
@@ -568,6 +644,32 @@ define amdgpu_kernel void @global_agent_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("agent") seq_cst, align 4
@@ -694,6 +796,30 @@ define amdgpu_kernel void @global_agent_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("agent") unordered, align 4
@@ -819,6 +945,30 @@ define amdgpu_kernel void @global_agent_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
@@ -946,6 +1096,30 @@ define amdgpu_kernel void @global_agent_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("agent") release, align 4
@@ -1073,6 +1247,30 @@ define amdgpu_kernel void @global_agent_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
@@ -1198,6 +1396,26 @@ define amdgpu_kernel void @global_agent_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") monotonic
@@ -1344,6 +1562,26 @@ define amdgpu_kernel void @global_agent_acquire_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") acquire
@@ -1471,6 +1709,26 @@ define amdgpu_kernel void @global_agent_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") release
@@ -1619,6 +1877,26 @@ define amdgpu_kernel void @global_agent_acq_rel_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") acq_rel
@@ -1767,6 +2045,26 @@ define amdgpu_kernel void @global_agent_seq_cst_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
@@ -1928,6 +2226,32 @@ define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") acquire
@@ -2092,6 +2416,32 @@ define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") acq_rel
@@ -2256,6 +2606,32 @@ define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
@@ -2380,6 +2756,28 @@ define amdgpu_kernel void @global_agent_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2525,6 +2923,28 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2651,6 +3071,28 @@ define amdgpu_kernel void @global_agent_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2798,6 +3240,28 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2945,6 +3409,28 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3090,6 +3576,28 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3235,6 +3743,28 @@ define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3382,6 +3912,28 @@ define amdgpu_kernel void @global_agent_release_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3529,6 +4081,28 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3676,6 +4250,28 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3823,6 +4419,28 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3970,6 +4588,28 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4117,6 +4757,28 @@ define amdgpu_kernel void @global_agent_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4264,6 +4926,28 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4411,6 +5095,28 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4559,6 +5265,32 @@ define amdgpu_kernel void @global_agent_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4723,6 +5455,34 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4875,6 +5635,32 @@ define amdgpu_kernel void @global_agent_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5041,6 +5827,34 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5207,6 +6021,34 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5371,6 +6213,34 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5535,6 +6405,34 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5701,6 +6599,34 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5867,6 +6793,34 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6033,6 +6987,34 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6199,6 +7181,34 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6365,6 +7375,34 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6531,6 +7569,34 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6697,6 +7763,34 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6863,6 +7957,34 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6999,6 +8121,30 @@ define amdgpu_kernel void @global_agent_one_as_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("agent-one-as") unordered, align 4
@@ -7133,6 +8279,30 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("agent-one-as") monotonic, align 4
@@ -7281,6 +8451,32 @@ define amdgpu_kernel void @global_agent_one_as_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("agent-one-as") acquire, align 4
@@ -7429,6 +8625,32 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("agent-one-as") seq_cst, align 4
@@ -7555,6 +8777,30 @@ define amdgpu_kernel void @global_agent_one_as_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("agent-one-as") unordered, align 4
@@ -7680,6 +8926,30 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("agent-one-as") monotonic, align 4
@@ -7807,6 +9077,30 @@ define amdgpu_kernel void @global_agent_one_as_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("agent-one-as") release, align 4
@@ -7934,6 +9228,30 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("agent-one-as") seq_cst, align 4
@@ -8059,6 +9377,26 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") monotonic
@@ -8205,6 +9543,26 @@ define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") acquire
@@ -8332,6 +9690,26 @@ define amdgpu_kernel void @global_agent_one_as_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") release
@@ -8480,6 +9858,26 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -8628,6 +10026,26 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -8789,6 +10207,32 @@ define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") acquire
@@ -8953,6 +10397,32 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -9117,6 +10587,32 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -9241,6 +10737,28 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9386,6 +10904,28 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9512,6 +11052,28 @@ define amdgpu_kernel void @global_agent_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9659,6 +11221,28 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9806,6 +11390,28 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9951,6 +11557,28 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10096,6 +11724,28 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10243,6 +11893,28 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10390,6 +12062,28 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10537,6 +12231,28 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10684,6 +12400,28 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10831,6 +12569,28 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10978,6 +12738,28 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11125,6 +12907,28 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11272,6 +13076,28 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11420,6 +13246,32 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11584,6 +13436,34 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11750,6 +13630,34 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11916,6 +13824,34 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12080,6 +14016,34 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12244,6 +14208,34 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12410,6 +14402,34 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12576,6 +14596,34 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12742,6 +14790,34 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12908,6 +14984,34 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13074,6 +15178,34 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13240,6 +15372,34 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13406,6 +15566,34 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13572,6 +15760,34 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index 07a3f85066991b..b2751326fa784b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @global_nontemporal_load_0(
; GFX6-LABEL: global_nontemporal_load_0:
@@ -141,6 +143,30 @@ define amdgpu_kernel void @global_nontemporal_load_0(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_nontemporal_load_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_nontemporal_load_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load i32, ptr addrspace(1) %in, align 4, !nontemporal !0
@@ -287,6 +313,30 @@ define amdgpu_kernel void @global_nontemporal_load_1(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_nontemporal_load_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v0, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_nontemporal_load_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v0, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -426,6 +476,30 @@ define amdgpu_kernel void @global_nontemporal_store_0(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_nontemporal_store_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3] th:TH_STORE_NT
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_nontemporal_store_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3] th:TH_STORE_NT
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load i32, ptr addrspace(1) %in, align 4
@@ -567,6 +641,30 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_nontemporal_store_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3] th:TH_STORE_NT
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_nontemporal_store_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3] th:TH_STORE_NT
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
index 495449ed8eee12..98cd97d07d6ff5 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @global_singlethread_unordered_load(
; GFX6-LABEL: global_singlethread_unordered_load:
@@ -138,6 +140,30 @@ define amdgpu_kernel void @global_singlethread_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread") unordered, align 4
@@ -272,6 +298,30 @@ define amdgpu_kernel void @global_singlethread_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread") monotonic, align 4
@@ -406,6 +456,30 @@ define amdgpu_kernel void @global_singlethread_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread") acquire, align 4
@@ -540,6 +614,30 @@ define amdgpu_kernel void @global_singlethread_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread") seq_cst, align 4
@@ -666,6 +764,30 @@ define amdgpu_kernel void @global_singlethread_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread") unordered, align 4
@@ -791,6 +913,30 @@ define amdgpu_kernel void @global_singlethread_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread") monotonic, align 4
@@ -916,6 +1062,30 @@ define amdgpu_kernel void @global_singlethread_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread") release, align 4
@@ -1041,6 +1211,30 @@ define amdgpu_kernel void @global_singlethread_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread") seq_cst, align 4
@@ -1166,6 +1360,26 @@ define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") monotonic
@@ -1291,6 +1505,26 @@ define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") acquire
@@ -1416,6 +1650,26 @@ define amdgpu_kernel void @global_singlethread_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") release
@@ -1541,6 +1795,26 @@ define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") acq_rel
@@ -1666,6 +1940,26 @@ define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") seq_cst
@@ -1813,6 +2107,30 @@ define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") acquire
@@ -1961,6 +2279,30 @@ define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") acq_rel
@@ -2109,6 +2451,30 @@ define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") seq_cst
@@ -2233,6 +2599,28 @@ define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2357,6 +2745,28 @@ define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2481,6 +2891,28 @@ define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2605,6 +3037,28 @@ define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2729,6 +3183,28 @@ define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2853,6 +3329,28 @@ define amdgpu_kernel void @global_singlethread_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2977,6 +3475,28 @@ define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3101,6 +3621,28 @@ define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3225,6 +3767,28 @@ define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3349,6 +3913,28 @@ define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3473,6 +4059,28 @@ define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3597,6 +4205,28 @@ define amdgpu_kernel void @global_singlethread_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3721,6 +4351,28 @@ define amdgpu_kernel void @global_singlethread_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3845,6 +4497,28 @@ define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3969,6 +4643,28 @@ define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4117,6 +4813,32 @@ define amdgpu_kernel void @global_singlethread_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4267,6 +4989,32 @@ define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4417,6 +5165,32 @@ define amdgpu_kernel void @global_singlethread_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4567,6 +5341,32 @@ define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4717,6 +5517,32 @@ define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4867,6 +5693,32 @@ define amdgpu_kernel void @global_singlethread_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5017,6 +5869,32 @@ define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5167,6 +6045,32 @@ define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5317,6 +6221,32 @@ define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5467,6 +6397,32 @@ define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5617,6 +6573,32 @@ define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5767,6 +6749,32 @@ define amdgpu_kernel void @global_singlethread_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5917,6 +6925,32 @@ define amdgpu_kernel void @global_singlethread_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6067,6 +7101,32 @@ define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6217,6 +7277,32 @@ define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6353,6 +7439,30 @@ define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread-one-as") unordered, align 4
@@ -6487,6 +7597,30 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread-one-as") monotonic, align 4
@@ -6621,6 +7755,30 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread-one-as") acquire, align 4
@@ -6755,6 +7913,30 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread-one-as") seq_cst, align 4
@@ -6881,6 +8063,30 @@ define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread-one-as") unordered, align 4
@@ -7006,6 +8212,30 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread-one-as") monotonic, align 4
@@ -7131,6 +8361,30 @@ define amdgpu_kernel void @global_singlethread_one_as_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread-one-as") release, align 4
@@ -7256,6 +8510,30 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread-one-as") seq_cst, align 4
@@ -7381,6 +8659,26 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") monotonic
@@ -7506,6 +8804,26 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -7631,6 +8949,26 @@ define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") release
@@ -7756,6 +9094,26 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -7881,6 +9239,26 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -8028,6 +9406,30 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -8176,6 +9578,30 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -8324,6 +9750,30 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -8448,6 +9898,28 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8572,6 +10044,28 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8696,6 +10190,28 @@ define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8820,6 +10336,28 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8944,6 +10482,28 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9068,6 +10628,28 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9192,6 +10774,28 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9316,6 +10920,28 @@ define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9440,6 +11066,28 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9564,6 +11212,28 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9688,6 +11358,28 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9812,6 +11504,28 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9936,6 +11650,28 @@ define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10060,6 +11796,28 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10184,6 +11942,28 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10332,6 +12112,32 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_ret_cm
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10482,6 +12288,32 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpx
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10632,6 +12464,32 @@ define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_ret_cmpx
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10782,6 +12640,32 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpx
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10932,6 +12816,32 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpx
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11082,6 +12992,32 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_ret_cmpx
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11232,6 +13168,32 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11382,6 +13344,32 @@ define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11532,6 +13520,32 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11682,6 +13696,32 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11832,6 +13872,32 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_ret_cmpx
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11982,6 +14048,32 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_ret_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12132,6 +14224,32 @@ define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_ret_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12282,6 +14400,32 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12432,6 +14576,32 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxch
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
index f611003a7ba531..a68a6a113830dc 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @global_system_unordered_load(
; GFX6-LABEL: global_system_unordered_load:
@@ -138,6 +140,30 @@ define amdgpu_kernel void @global_system_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in unordered, align 4
@@ -272,6 +298,30 @@ define amdgpu_kernel void @global_system_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in monotonic, align 4
@@ -422,6 +472,32 @@ define amdgpu_kernel void @global_system_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in acquire, align 4
@@ -572,6 +648,32 @@ define amdgpu_kernel void @global_system_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in seq_cst, align 4
@@ -698,6 +800,30 @@ define amdgpu_kernel void @global_system_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out unordered, align 4
@@ -823,6 +949,30 @@ define amdgpu_kernel void @global_system_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out monotonic, align 4
@@ -952,6 +1102,30 @@ define amdgpu_kernel void @global_system_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out release, align 4
@@ -1081,6 +1255,30 @@ define amdgpu_kernel void @global_system_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out seq_cst, align 4
@@ -1206,6 +1404,26 @@ define amdgpu_kernel void @global_system_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in monotonic
@@ -1354,6 +1572,26 @@ define amdgpu_kernel void @global_system_acquire_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in acquire
@@ -1483,6 +1721,26 @@ define amdgpu_kernel void @global_system_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in release
@@ -1635,6 +1893,26 @@ define amdgpu_kernel void @global_system_acq_rel_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in acq_rel
@@ -1787,6 +2065,26 @@ define amdgpu_kernel void @global_system_seq_cst_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in seq_cst
@@ -1950,6 +2248,32 @@ define amdgpu_kernel void @global_system_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in acquire
@@ -2118,6 +2442,32 @@ define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in acq_rel
@@ -2286,6 +2636,32 @@ define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in seq_cst
@@ -2410,6 +2786,28 @@ define amdgpu_kernel void @global_system_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2557,6 +2955,28 @@ define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2685,6 +3105,28 @@ define amdgpu_kernel void @global_system_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2836,6 +3278,28 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2987,6 +3451,28 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3134,6 +3620,28 @@ define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3281,6 +3789,28 @@ define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3432,6 +3962,28 @@ define amdgpu_kernel void @global_system_release_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3583,6 +4135,28 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3734,6 +4308,28 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3885,6 +4481,28 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4033,6 +4651,32 @@ define amdgpu_kernel void @global_system_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4199,6 +4843,34 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4369,6 +5041,34 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4539,6 +5239,34 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4705,6 +5433,34 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4871,6 +5627,34 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5041,6 +5825,34 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5211,6 +6023,34 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5381,6 +6221,34 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5551,6 +6419,34 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5721,6 +6617,34 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5891,6 +6815,34 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6061,6 +7013,34 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6231,6 +7211,34 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6367,6 +7375,30 @@ define amdgpu_kernel void @global_system_one_as_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("one-as") unordered, align 4
@@ -6501,6 +7533,30 @@ define amdgpu_kernel void @global_system_one_as_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("one-as") monotonic, align 4
@@ -6651,6 +7707,32 @@ define amdgpu_kernel void @global_system_one_as_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("one-as") acquire, align 4
@@ -6801,6 +7883,32 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("one-as") seq_cst, align 4
@@ -6927,6 +8035,30 @@ define amdgpu_kernel void @global_system_one_as_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("one-as") unordered, align 4
@@ -7052,6 +8184,30 @@ define amdgpu_kernel void @global_system_one_as_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("one-as") monotonic, align 4
@@ -7181,6 +8337,30 @@ define amdgpu_kernel void @global_system_one_as_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("one-as") release, align 4
@@ -7310,6 +8490,30 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("one-as") seq_cst, align 4
@@ -7435,6 +8639,26 @@ define amdgpu_kernel void @global_system_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") monotonic
@@ -7583,6 +8807,26 @@ define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") acquire
@@ -7712,6 +8956,26 @@ define amdgpu_kernel void @global_system_one_as_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") release
@@ -7864,6 +9128,26 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") acq_rel
@@ -8016,6 +9300,26 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") seq_cst
@@ -8179,6 +9483,32 @@ define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") acquire
@@ -8347,6 +9677,32 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") acq_rel
@@ -8515,6 +9871,32 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") seq_cst
@@ -8639,6 +10021,28 @@ define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8786,6 +10190,28 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8914,6 +10340,28 @@ define amdgpu_kernel void @global_system_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9065,6 +10513,28 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9216,6 +10686,28 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9363,6 +10855,28 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9510,6 +11024,28 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9661,6 +11197,28 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9812,6 +11370,28 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9963,6 +11543,28 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10114,6 +11716,28 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10265,6 +11889,28 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10416,6 +12062,28 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10567,6 +12235,28 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10718,6 +12408,28 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: buffer_gl1_inv
; GFX11-CU-NEXT: buffer_gl0_inv
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10866,6 +12578,32 @@ define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11032,6 +12770,34 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11186,6 +12952,32 @@ define amdgpu_kernel void @global_system_one_as_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11356,6 +13148,34 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11526,6 +13346,34 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11692,6 +13540,34 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11858,6 +13734,34 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12028,6 +13932,34 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12198,6 +14130,34 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12368,6 +14328,34 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12538,6 +14526,34 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12708,6 +14724,34 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12878,6 +14922,34 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13048,6 +15120,34 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13218,6 +15318,34 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
index 5875420eac09a4..263324ed5c03b4 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
@@ -6,6 +6,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @global_volatile_load_0(
; GFX6-LABEL: global_volatile_load_0:
@@ -94,6 +96,30 @@ define amdgpu_kernel void @global_volatile_load_0(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_volatile_load_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_volatile_load_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load volatile i32, ptr addrspace(1) %in, align 4
@@ -196,6 +222,30 @@ define amdgpu_kernel void @global_volatile_load_1(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_volatile_load_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_volatile_load_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -298,6 +348,32 @@ define amdgpu_kernel void @global_volatile_store_0(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_volatile_store_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_volatile_store_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load i32, ptr addrspace(1) %in, align 4
@@ -402,6 +478,32 @@ define amdgpu_kernel void @global_volatile_store_1(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_volatile_store_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_volatile_store_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_storecnt 0x0
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -500,6 +602,31 @@ define amdgpu_kernel void @global_volatile_workgroup_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_volatile_workgroup_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_volatile_workgroup_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic volatile i32, ptr addrspace(1) %in syncscope("workgroup") acquire, align 4
@@ -586,6 +713,30 @@ define amdgpu_kernel void @global_volatile_workgroup_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_volatile_workgroup_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_volatile_workgroup_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic volatile i32 %in, ptr addrspace(1) %out syncscope("workgroup") release, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
index db1877af5541c8..9683530631767c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @global_wavefront_unordered_load(
; GFX6-LABEL: global_wavefront_unordered_load:
@@ -138,6 +140,30 @@ define amdgpu_kernel void @global_wavefront_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront") unordered, align 4
@@ -272,6 +298,30 @@ define amdgpu_kernel void @global_wavefront_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront") monotonic, align 4
@@ -406,6 +456,30 @@ define amdgpu_kernel void @global_wavefront_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront") acquire, align 4
@@ -540,6 +614,30 @@ define amdgpu_kernel void @global_wavefront_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront") seq_cst, align 4
@@ -666,6 +764,30 @@ define amdgpu_kernel void @global_wavefront_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront") unordered, align 4
@@ -791,6 +913,30 @@ define amdgpu_kernel void @global_wavefront_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
@@ -916,6 +1062,30 @@ define amdgpu_kernel void @global_wavefront_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront") release, align 4
@@ -1041,6 +1211,30 @@ define amdgpu_kernel void @global_wavefront_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 4
@@ -1166,6 +1360,26 @@ define amdgpu_kernel void @global_wavefront_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") monotonic
@@ -1291,6 +1505,26 @@ define amdgpu_kernel void @global_wavefront_acquire_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") acquire
@@ -1416,6 +1650,26 @@ define amdgpu_kernel void @global_wavefront_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") release
@@ -1541,6 +1795,26 @@ define amdgpu_kernel void @global_wavefront_acq_rel_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") acq_rel
@@ -1666,6 +1940,26 @@ define amdgpu_kernel void @global_wavefront_seq_cst_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") seq_cst
@@ -1813,6 +2107,30 @@ define amdgpu_kernel void @global_wavefront_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") acquire
@@ -1961,6 +2279,30 @@ define amdgpu_kernel void @global_wavefront_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") acq_rel
@@ -2109,6 +2451,30 @@ define amdgpu_kernel void @global_wavefront_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") seq_cst
@@ -2233,6 +2599,28 @@ define amdgpu_kernel void @global_wavefront_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2357,6 +2745,28 @@ define amdgpu_kernel void @global_wavefront_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2481,6 +2891,28 @@ define amdgpu_kernel void @global_wavefront_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2605,6 +3037,28 @@ define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2729,6 +3183,28 @@ define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2853,6 +3329,28 @@ define amdgpu_kernel void @global_wavefront_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2977,6 +3475,28 @@ define amdgpu_kernel void @global_wavefront_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3101,6 +3621,28 @@ define amdgpu_kernel void @global_wavefront_release_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3225,6 +3767,28 @@ define amdgpu_kernel void @global_wavefront_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3349,6 +3913,28 @@ define amdgpu_kernel void @global_wavefront_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3473,6 +4059,28 @@ define amdgpu_kernel void @global_wavefront_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3597,6 +4205,28 @@ define amdgpu_kernel void @global_wavefront_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3721,6 +4351,28 @@ define amdgpu_kernel void @global_wavefront_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3845,6 +4497,28 @@ define amdgpu_kernel void @global_wavefront_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3969,6 +4643,28 @@ define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4117,6 +4813,32 @@ define amdgpu_kernel void @global_wavefront_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4267,6 +4989,32 @@ define amdgpu_kernel void @global_wavefront_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4417,6 +5165,32 @@ define amdgpu_kernel void @global_wavefront_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4567,6 +5341,32 @@ define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4717,6 +5517,32 @@ define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4867,6 +5693,32 @@ define amdgpu_kernel void @global_wavefront_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5017,6 +5869,32 @@ define amdgpu_kernel void @global_wavefront_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5167,6 +6045,32 @@ define amdgpu_kernel void @global_wavefront_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5317,6 +6221,32 @@ define amdgpu_kernel void @global_wavefront_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5467,6 +6397,32 @@ define amdgpu_kernel void @global_wavefront_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5617,6 +6573,32 @@ define amdgpu_kernel void @global_wavefront_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5767,6 +6749,32 @@ define amdgpu_kernel void @global_wavefront_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5917,6 +6925,32 @@ define amdgpu_kernel void @global_wavefront_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6067,6 +7101,32 @@ define amdgpu_kernel void @global_wavefront_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6217,6 +7277,32 @@ define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6353,6 +7439,30 @@ define amdgpu_kernel void @global_wavefront_one_as_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront-one-as") unordered, align 4
@@ -6487,6 +7597,30 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront-one-as") monotonic, align 4
@@ -6621,6 +7755,30 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront-one-as") acquire, align 4
@@ -6755,6 +7913,30 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront-one-as") seq_cst, align 4
@@ -6881,6 +8063,30 @@ define amdgpu_kernel void @global_wavefront_one_as_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront-one-as") unordered, align 4
@@ -7006,6 +8212,30 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront-one-as") monotonic, align 4
@@ -7131,6 +8361,30 @@ define amdgpu_kernel void @global_wavefront_one_as_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront-one-as") release, align 4
@@ -7256,6 +8510,30 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront-one-as") seq_cst, align 4
@@ -7381,6 +8659,26 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") monotonic
@@ -7506,6 +8804,26 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -7631,6 +8949,26 @@ define amdgpu_kernel void @global_wavefront_one_as_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") release
@@ -7756,6 +9094,26 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -7881,6 +9239,26 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -8028,6 +9406,30 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -8176,6 +9578,30 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -8324,6 +9750,30 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -8448,6 +9898,28 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8572,6 +10044,28 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8696,6 +10190,28 @@ define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8820,6 +10336,28 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8944,6 +10482,28 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9068,6 +10628,28 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9192,6 +10774,28 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9316,6 +10920,28 @@ define amdgpu_kernel void @global_wavefront_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9440,6 +11066,28 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9564,6 +11212,28 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9688,6 +11358,28 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9812,6 +11504,28 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9936,6 +11650,28 @@ define amdgpu_kernel void @global_wavefront_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10060,6 +11796,28 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10184,6 +11942,28 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10332,6 +12112,32 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_ret_cmpxc
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10482,6 +12288,32 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10632,6 +12464,32 @@ define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10782,6 +12640,32 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10932,6 +12816,32 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11082,6 +12992,32 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_acquire_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11232,6 +13168,32 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11382,6 +13344,32 @@ define amdgpu_kernel void @global_wavefront_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11532,6 +13520,32 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11682,6 +13696,32 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11832,6 +13872,32 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11982,6 +14048,32 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12132,6 +14224,32 @@ define amdgpu_kernel void @global_wavefront_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12282,6 +14400,32 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12432,6 +14576,32 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
index 0db3cfb36bb7fd..550f06b970bcd9 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @global_workgroup_unordered_load(
; GFX6-LABEL: global_workgroup_unordered_load:
@@ -138,6 +140,30 @@ define amdgpu_kernel void @global_workgroup_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") unordered, align 4
@@ -272,6 +298,30 @@ define amdgpu_kernel void @global_workgroup_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") monotonic, align 4
@@ -410,6 +460,31 @@ define amdgpu_kernel void @global_workgroup_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") acquire, align 4
@@ -548,6 +623,31 @@ define amdgpu_kernel void @global_workgroup_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") seq_cst, align 4
@@ -674,6 +774,30 @@ define amdgpu_kernel void @global_workgroup_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") unordered, align 4
@@ -799,6 +923,30 @@ define amdgpu_kernel void @global_workgroup_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") monotonic, align 4
@@ -924,6 +1072,30 @@ define amdgpu_kernel void @global_workgroup_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") release, align 4
@@ -1049,6 +1221,30 @@ define amdgpu_kernel void @global_workgroup_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") seq_cst, align 4
@@ -1174,6 +1370,26 @@ define amdgpu_kernel void @global_workgroup_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") monotonic
@@ -1305,6 +1521,26 @@ define amdgpu_kernel void @global_workgroup_acquire_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acquire
@@ -1430,6 +1666,26 @@ define amdgpu_kernel void @global_workgroup_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") release
@@ -1561,6 +1817,26 @@ define amdgpu_kernel void @global_workgroup_acq_rel_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acq_rel
@@ -1692,6 +1968,26 @@ define amdgpu_kernel void @global_workgroup_seq_cst_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
@@ -1843,6 +2139,31 @@ define amdgpu_kernel void @global_workgroup_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acquire
@@ -1995,6 +2316,31 @@ define amdgpu_kernel void @global_workgroup_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acq_rel
@@ -2147,6 +2493,31 @@ define amdgpu_kernel void @global_workgroup_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
@@ -2271,6 +2642,28 @@ define amdgpu_kernel void @global_workgroup_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2401,6 +2794,28 @@ define amdgpu_kernel void @global_workgroup_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2525,6 +2940,28 @@ define amdgpu_kernel void @global_workgroup_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2655,6 +3092,28 @@ define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2785,6 +3244,28 @@ define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -2915,6 +3396,28 @@ define amdgpu_kernel void @global_workgroup_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3045,6 +3548,28 @@ define amdgpu_kernel void @global_workgroup_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3175,6 +3700,28 @@ define amdgpu_kernel void @global_workgroup_release_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3305,6 +3852,28 @@ define amdgpu_kernel void @global_workgroup_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3435,6 +4004,28 @@ define amdgpu_kernel void @global_workgroup_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3565,6 +4156,28 @@ define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3695,6 +4308,28 @@ define amdgpu_kernel void @global_workgroup_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3825,6 +4460,28 @@ define amdgpu_kernel void @global_workgroup_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3955,6 +4612,28 @@ define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4085,6 +4764,28 @@ define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4233,6 +4934,32 @@ define amdgpu_kernel void @global_workgroup_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4387,6 +5114,33 @@ define amdgpu_kernel void @global_workgroup_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4537,6 +5291,32 @@ define amdgpu_kernel void @global_workgroup_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4691,6 +5471,33 @@ define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4845,6 +5652,33 @@ define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4999,6 +5833,33 @@ define amdgpu_kernel void @global_workgroup_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5153,6 +6014,33 @@ define amdgpu_kernel void @global_workgroup_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5307,6 +6195,33 @@ define amdgpu_kernel void @global_workgroup_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5461,6 +6376,33 @@ define amdgpu_kernel void @global_workgroup_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5615,6 +6557,33 @@ define amdgpu_kernel void @global_workgroup_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5769,6 +6738,33 @@ define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5923,6 +6919,33 @@ define amdgpu_kernel void @global_workgroup_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6077,6 +7100,33 @@ define amdgpu_kernel void @global_workgroup_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6231,6 +7281,33 @@ define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6385,6 +7462,33 @@ define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6521,6 +7625,30 @@ define amdgpu_kernel void @global_workgroup_one_as_unordered_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") unordered, align 4
@@ -6655,6 +7783,30 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") monotonic, align 4
@@ -6793,6 +7945,31 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") acquire, align 4
@@ -6931,6 +8108,31 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(1) %out) {
entry:
%val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") seq_cst, align 4
@@ -7057,6 +8259,30 @@ define amdgpu_kernel void @global_workgroup_one_as_unordered_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") unordered, align 4
@@ -7182,6 +8408,30 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") monotonic, align 4
@@ -7307,6 +8557,30 @@ define amdgpu_kernel void @global_workgroup_one_as_release_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") release, align 4
@@ -7432,6 +8706,30 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_store(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(1) %out) {
entry:
store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") seq_cst, align 4
@@ -7557,6 +8855,26 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") monotonic
@@ -7688,6 +9006,26 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -7813,6 +9151,26 @@ define amdgpu_kernel void @global_workgroup_one_as_release_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") release
@@ -7944,6 +9302,26 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -8075,6 +9453,26 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -8226,6 +9624,31 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -8378,6 +9801,31 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -8530,6 +9978,31 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -8654,6 +10127,28 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8784,6 +10279,28 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8908,6 +10425,28 @@ define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9038,6 +10577,28 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9168,6 +10729,28 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9298,6 +10881,28 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9428,6 +11033,28 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9558,6 +11185,28 @@ define amdgpu_kernel void @global_workgroup_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9688,6 +11337,28 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9818,6 +11489,28 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9948,6 +11641,28 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10078,6 +11793,28 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10208,6 +11945,28 @@ define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10338,6 +12097,28 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10468,6 +12249,28 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-WGP-NEXT: s_wait_storecnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10616,6 +12419,32 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_ret_cmpxc
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10770,6 +12599,33 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10920,6 +12776,32 @@ define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11074,6 +12956,33 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11228,6 +13137,33 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11382,6 +13318,33 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11536,6 +13499,33 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11690,6 +13680,33 @@ define amdgpu_kernel void @global_workgroup_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11844,6 +13861,33 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11998,6 +14042,33 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12152,6 +14223,33 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12306,6 +14404,33 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12460,6 +14585,33 @@ define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12614,6 +14766,33 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12768,6 +14947,33 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
index b53065cf897864..1e3333c4287e20 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @local_agent_unordered_load(
; GFX6-LABEL: local_agent_unordered_load:
@@ -133,6 +135,26 @@ define amdgpu_kernel void @local_agent_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("agent") unordered, align 4
@@ -262,6 +284,26 @@ define amdgpu_kernel void @local_agent_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("agent") monotonic, align 4
@@ -395,6 +437,27 @@ define amdgpu_kernel void @local_agent_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("agent") acquire, align 4
@@ -528,6 +591,27 @@ define amdgpu_kernel void @local_agent_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("agent") seq_cst, align 4
@@ -635,6 +719,22 @@ define amdgpu_kernel void @local_agent_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("agent") unordered, align 4
@@ -741,6 +841,22 @@ define amdgpu_kernel void @local_agent_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 4
@@ -847,6 +963,22 @@ define amdgpu_kernel void @local_agent_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("agent") release, align 4
@@ -953,6 +1085,22 @@ define amdgpu_kernel void @local_agent_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 4
@@ -1059,6 +1207,22 @@ define amdgpu_kernel void @local_agent_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") monotonic
@@ -1178,6 +1342,25 @@ define amdgpu_kernel void @local_agent_acquire_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") acquire
@@ -1284,6 +1467,22 @@ define amdgpu_kernel void @local_agent_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") release
@@ -1403,6 +1602,25 @@ define amdgpu_kernel void @local_agent_acq_rel_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") acq_rel
@@ -1522,6 +1740,25 @@ define amdgpu_kernel void @local_agent_seq_cst_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") seq_cst
@@ -1654,6 +1891,27 @@ define amdgpu_kernel void @local_agent_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") acquire
@@ -1787,6 +2045,27 @@ define amdgpu_kernel void @local_agent_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") acq_rel
@@ -1920,6 +2199,27 @@ define amdgpu_kernel void @local_agent_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") seq_cst
@@ -2039,6 +2339,24 @@ define amdgpu_kernel void @local_agent_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2171,6 +2489,27 @@ define amdgpu_kernel void @local_agent_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2290,6 +2629,24 @@ define amdgpu_kernel void @local_agent_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2422,6 +2779,27 @@ define amdgpu_kernel void @local_agent_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2554,6 +2932,27 @@ define amdgpu_kernel void @local_agent_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2686,6 +3085,27 @@ define amdgpu_kernel void @local_agent_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2818,6 +3238,27 @@ define amdgpu_kernel void @local_agent_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2950,6 +3391,27 @@ define amdgpu_kernel void @local_agent_release_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3082,6 +3544,27 @@ define amdgpu_kernel void @local_agent_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3214,6 +3697,27 @@ define amdgpu_kernel void @local_agent_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3346,6 +3850,27 @@ define amdgpu_kernel void @local_agent_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3478,6 +4003,27 @@ define amdgpu_kernel void @local_agent_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3610,6 +4156,27 @@ define amdgpu_kernel void @local_agent_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3742,6 +4309,27 @@ define amdgpu_kernel void @local_agent_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3874,6 +4462,27 @@ define amdgpu_kernel void @local_agent_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4015,6 +4624,28 @@ define amdgpu_kernel void @local_agent_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4162,6 +4793,29 @@ define amdgpu_kernel void @local_agent_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4305,6 +4959,28 @@ define amdgpu_kernel void @local_agent_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4452,6 +5128,29 @@ define amdgpu_kernel void @local_agent_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4599,6 +5298,29 @@ define amdgpu_kernel void @local_agent_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4746,6 +5468,29 @@ define amdgpu_kernel void @local_agent_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4893,6 +5638,29 @@ define amdgpu_kernel void @local_agent_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5040,6 +5808,29 @@ define amdgpu_kernel void @local_agent_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5187,6 +5978,29 @@ define amdgpu_kernel void @local_agent_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5334,6 +6148,29 @@ define amdgpu_kernel void @local_agent_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5481,6 +6318,29 @@ define amdgpu_kernel void @local_agent_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5628,6 +6488,29 @@ define amdgpu_kernel void @local_agent_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5775,6 +6658,29 @@ define amdgpu_kernel void @local_agent_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5922,6 +6828,29 @@ define amdgpu_kernel void @local_agent_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6069,6 +6998,29 @@ define amdgpu_kernel void @local_agent_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6200,6 +7152,26 @@ define amdgpu_kernel void @local_agent_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("agent-one-as") unordered, align 4
@@ -6329,6 +7301,26 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("agent-one-as") monotonic, align 4
@@ -6458,6 +7450,26 @@ define amdgpu_kernel void @local_agent_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("agent-one-as") acquire, align 4
@@ -6587,6 +7599,26 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("agent-one-as") seq_cst, align 4
@@ -6694,6 +7726,22 @@ define amdgpu_kernel void @local_agent_one_as_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("agent-one-as") unordered, align 4
@@ -6800,6 +7848,22 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("agent-one-as") monotonic, align 4
@@ -6906,6 +7970,22 @@ define amdgpu_kernel void @local_agent_one_as_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("agent-one-as") release, align 4
@@ -7012,6 +8092,22 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("agent-one-as") seq_cst, align 4
@@ -7118,6 +8214,22 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") monotonic
@@ -7224,6 +8336,22 @@ define amdgpu_kernel void @local_agent_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") acquire
@@ -7330,6 +8458,22 @@ define amdgpu_kernel void @local_agent_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") release
@@ -7436,6 +8580,22 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -7542,6 +8702,22 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -7670,6 +8846,26 @@ define amdgpu_kernel void @local_agent_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") acquire
@@ -7799,6 +8995,26 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -7928,6 +9144,26 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -8047,6 +9283,24 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8166,6 +9420,24 @@ define amdgpu_kernel void @local_agent_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8285,6 +9557,24 @@ define amdgpu_kernel void @local_agent_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8404,6 +9694,24 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8523,6 +9831,24 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8642,6 +9968,24 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8761,6 +10105,24 @@ define amdgpu_kernel void @local_agent_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8880,6 +10242,24 @@ define amdgpu_kernel void @local_agent_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8999,6 +10379,24 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9118,6 +10516,24 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9237,6 +10653,24 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9356,6 +10790,24 @@ define amdgpu_kernel void @local_agent_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9475,6 +10927,24 @@ define amdgpu_kernel void @local_agent_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9594,6 +11064,24 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9713,6 +11201,24 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9854,6 +11360,28 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9997,6 +11525,28 @@ define amdgpu_kernel void @local_agent_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10140,6 +11690,28 @@ define amdgpu_kernel void @local_agent_one_as_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10283,6 +11855,28 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10426,6 +12020,28 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10569,6 +12185,28 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10712,6 +12350,28 @@ define amdgpu_kernel void @local_agent_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10855,6 +12515,28 @@ define amdgpu_kernel void @local_agent_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10998,6 +12680,28 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11141,6 +12845,28 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11284,6 +13010,28 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11427,6 +13175,28 @@ define amdgpu_kernel void @local_agent_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11570,6 +13340,28 @@ define amdgpu_kernel void @local_agent_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11713,6 +13505,28 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11856,6 +13670,28 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
index 70c91d26006026..0eb2f7b1d694ec 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @local_nontemporal_load_0(
; GFX6-LABEL: local_nontemporal_load_0:
@@ -155,6 +157,34 @@ define amdgpu_kernel void @local_nontemporal_load_0(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_nontemporal_load_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_nontemporal_load_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(1) %out) {
entry:
%val = load i32, ptr addrspace(3) %in, align 4, !nontemporal !0
@@ -311,6 +341,36 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_nontemporal_load_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s2
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_nontemporal_load_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s2
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(1) %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -457,6 +517,26 @@ define amdgpu_kernel void @local_nontemporal_store_0(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_nontemporal_store_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_nontemporal_store_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(3) %out) {
entry:
%val = load i32, ptr addrspace(1) %in, align 4
@@ -606,6 +686,28 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_nontemporal_store_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s2
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_nontemporal_store_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s2
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(3) %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
index 1f494bf3d36053..664203a8f7305b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @local_singlethread_unordered_load(
; GFX6-LABEL: local_singlethread_unordered_load:
@@ -133,6 +135,26 @@ define amdgpu_kernel void @local_singlethread_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread") unordered, align 4
@@ -262,6 +284,26 @@ define amdgpu_kernel void @local_singlethread_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread") monotonic, align 4
@@ -391,6 +433,26 @@ define amdgpu_kernel void @local_singlethread_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread") acquire, align 4
@@ -520,6 +582,26 @@ define amdgpu_kernel void @local_singlethread_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread") seq_cst, align 4
@@ -627,6 +709,22 @@ define amdgpu_kernel void @local_singlethread_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread") unordered, align 4
@@ -733,6 +831,22 @@ define amdgpu_kernel void @local_singlethread_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread") monotonic, align 4
@@ -839,6 +953,22 @@ define amdgpu_kernel void @local_singlethread_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread") release, align 4
@@ -945,6 +1075,22 @@ define amdgpu_kernel void @local_singlethread_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread") seq_cst, align 4
@@ -1051,6 +1197,22 @@ define amdgpu_kernel void @local_singlethread_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") monotonic
@@ -1157,6 +1319,22 @@ define amdgpu_kernel void @local_singlethread_acquire_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") acquire
@@ -1263,6 +1441,22 @@ define amdgpu_kernel void @local_singlethread_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") release
@@ -1369,6 +1563,22 @@ define amdgpu_kernel void @local_singlethread_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") acq_rel
@@ -1475,6 +1685,22 @@ define amdgpu_kernel void @local_singlethread_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") seq_cst
@@ -1603,6 +1829,26 @@ define amdgpu_kernel void @local_singlethread_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") acquire
@@ -1732,6 +1978,26 @@ define amdgpu_kernel void @local_singlethread_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") acq_rel
@@ -1861,6 +2127,26 @@ define amdgpu_kernel void @local_singlethread_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") seq_cst
@@ -1980,6 +2266,24 @@ define amdgpu_kernel void @local_singlethread_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2099,6 +2403,24 @@ define amdgpu_kernel void @local_singlethread_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2218,6 +2540,24 @@ define amdgpu_kernel void @local_singlethread_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2337,6 +2677,24 @@ define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2456,6 +2814,24 @@ define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2575,6 +2951,24 @@ define amdgpu_kernel void @local_singlethread_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2694,6 +3088,24 @@ define amdgpu_kernel void @local_singlethread_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2813,6 +3225,24 @@ define amdgpu_kernel void @local_singlethread_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2932,6 +3362,24 @@ define amdgpu_kernel void @local_singlethread_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3051,6 +3499,24 @@ define amdgpu_kernel void @local_singlethread_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3170,6 +3636,24 @@ define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3289,6 +3773,24 @@ define amdgpu_kernel void @local_singlethread_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3408,6 +3910,24 @@ define amdgpu_kernel void @local_singlethread_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3527,6 +4047,24 @@ define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3646,6 +4184,24 @@ define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3787,6 +4343,28 @@ define amdgpu_kernel void @local_singlethread_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3930,6 +4508,28 @@ define amdgpu_kernel void @local_singlethread_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4073,6 +4673,28 @@ define amdgpu_kernel void @local_singlethread_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4216,6 +4838,28 @@ define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4359,6 +5003,28 @@ define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4502,6 +5168,28 @@ define amdgpu_kernel void @local_singlethread_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4645,6 +5333,28 @@ define amdgpu_kernel void @local_singlethread_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4788,6 +5498,28 @@ define amdgpu_kernel void @local_singlethread_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4931,6 +5663,28 @@ define amdgpu_kernel void @local_singlethread_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5074,6 +5828,28 @@ define amdgpu_kernel void @local_singlethread_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5217,6 +5993,28 @@ define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5360,6 +6158,28 @@ define amdgpu_kernel void @local_singlethread_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5503,6 +6323,28 @@ define amdgpu_kernel void @local_singlethread_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5646,6 +6488,28 @@ define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5789,6 +6653,28 @@ define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5920,6 +6806,26 @@ define amdgpu_kernel void @local_singlethread_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread-one-as") unordered, align 4
@@ -6049,6 +6955,26 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread-one-as") monotonic, align 4
@@ -6178,6 +7104,26 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread-one-as") acquire, align 4
@@ -6307,6 +7253,26 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread-one-as") seq_cst, align 4
@@ -6414,6 +7380,22 @@ define amdgpu_kernel void @local_singlethread_one_as_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread-one-as") unordered, align 4
@@ -6520,6 +7502,22 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread-one-as") monotonic, align 4
@@ -6626,6 +7624,22 @@ define amdgpu_kernel void @local_singlethread_one_as_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread-one-as") release, align 4
@@ -6732,6 +7746,22 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread-one-as") seq_cst, align 4
@@ -6838,6 +7868,22 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") monotonic
@@ -6944,6 +7990,22 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -7050,6 +8112,22 @@ define amdgpu_kernel void @local_singlethread_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") release
@@ -7156,6 +8234,22 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -7262,6 +8356,22 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -7390,6 +8500,26 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -7519,6 +8649,26 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -7648,6 +8798,26 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -7767,6 +8937,24 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_cmpxchg
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7886,6 +9074,24 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8005,6 +9211,24 @@ define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8124,6 +9348,24 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8243,6 +9485,24 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8362,6 +9622,24 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8481,6 +9759,24 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8600,6 +9896,24 @@ define amdgpu_kernel void @local_singlethread_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8719,6 +10033,24 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8838,6 +10170,24 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8957,6 +10307,24 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9076,6 +10444,24 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9195,6 +10581,24 @@ define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9314,6 +10718,24 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9433,6 +10855,24 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9574,6 +11014,28 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_ret_cmp
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9717,6 +11179,28 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_ret_cmpxc
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9860,6 +11344,28 @@ define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_ret_cmpxc
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10003,6 +11509,28 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_ret_cmpxc
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10146,6 +11674,28 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_ret_cmpxc
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10289,6 +11839,28 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_ret_cmpxc
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10432,6 +12004,28 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10575,6 +12169,28 @@ define amdgpu_kernel void @local_singlethread_one_as_release_acquire_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10718,6 +12334,28 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10861,6 +12499,28 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11004,6 +12664,28 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_ret_cmpxc
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11147,6 +12829,28 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11290,6 +12994,28 @@ define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11433,6 +13159,28 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11576,6 +13324,28 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
index baec9cb601bbd4..1a4268548b02b0 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @local_system_unordered_load(
; GFX6-LABEL: local_system_unordered_load:
@@ -133,6 +135,26 @@ define amdgpu_kernel void @local_system_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in unordered, align 4
@@ -262,6 +284,26 @@ define amdgpu_kernel void @local_system_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in monotonic, align 4
@@ -395,6 +437,27 @@ define amdgpu_kernel void @local_system_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in acquire, align 4
@@ -528,6 +591,27 @@ define amdgpu_kernel void @local_system_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in seq_cst, align 4
@@ -635,6 +719,22 @@ define amdgpu_kernel void @local_system_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out unordered, align 4
@@ -741,6 +841,22 @@ define amdgpu_kernel void @local_system_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out monotonic, align 4
@@ -847,6 +963,22 @@ define amdgpu_kernel void @local_system_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out release, align 4
@@ -953,6 +1085,22 @@ define amdgpu_kernel void @local_system_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out seq_cst, align 4
@@ -1059,6 +1207,22 @@ define amdgpu_kernel void @local_system_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in monotonic
@@ -1178,6 +1342,25 @@ define amdgpu_kernel void @local_system_acquire_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in acquire
@@ -1284,6 +1467,22 @@ define amdgpu_kernel void @local_system_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in release
@@ -1403,6 +1602,25 @@ define amdgpu_kernel void @local_system_acq_rel_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in acq_rel
@@ -1522,6 +1740,25 @@ define amdgpu_kernel void @local_system_seq_cst_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in seq_cst
@@ -1654,6 +1891,27 @@ define amdgpu_kernel void @local_system_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in acquire
@@ -1787,6 +2045,27 @@ define amdgpu_kernel void @local_system_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in acq_rel
@@ -1920,6 +2199,27 @@ define amdgpu_kernel void @local_system_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in seq_cst
@@ -2039,6 +2339,24 @@ define amdgpu_kernel void @local_system_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2171,6 +2489,27 @@ define amdgpu_kernel void @local_system_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2290,6 +2629,24 @@ define amdgpu_kernel void @local_system_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2422,6 +2779,27 @@ define amdgpu_kernel void @local_system_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2554,6 +2932,27 @@ define amdgpu_kernel void @local_system_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2686,6 +3085,27 @@ define amdgpu_kernel void @local_system_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2818,6 +3238,27 @@ define amdgpu_kernel void @local_system_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2950,6 +3391,27 @@ define amdgpu_kernel void @local_system_release_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3082,6 +3544,27 @@ define amdgpu_kernel void @local_system_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3214,6 +3697,27 @@ define amdgpu_kernel void @local_system_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3346,6 +3850,27 @@ define amdgpu_kernel void @local_system_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3478,6 +4003,27 @@ define amdgpu_kernel void @local_system_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3610,6 +4156,27 @@ define amdgpu_kernel void @local_system_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3742,6 +4309,27 @@ define amdgpu_kernel void @local_system_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3874,6 +4462,27 @@ define amdgpu_kernel void @local_system_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4015,6 +4624,28 @@ define amdgpu_kernel void @local_system_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4162,6 +4793,29 @@ define amdgpu_kernel void @local_system_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4305,6 +4959,28 @@ define amdgpu_kernel void @local_system_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4452,6 +5128,29 @@ define amdgpu_kernel void @local_system_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4599,6 +5298,29 @@ define amdgpu_kernel void @local_system_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4746,6 +5468,29 @@ define amdgpu_kernel void @local_system_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4893,6 +5638,29 @@ define amdgpu_kernel void @local_system_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5040,6 +5808,29 @@ define amdgpu_kernel void @local_system_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5187,6 +5978,29 @@ define amdgpu_kernel void @local_system_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5334,6 +6148,29 @@ define amdgpu_kernel void @local_system_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5481,6 +6318,29 @@ define amdgpu_kernel void @local_system_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5628,6 +6488,29 @@ define amdgpu_kernel void @local_system_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5775,6 +6658,29 @@ define amdgpu_kernel void @local_system_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5922,6 +6828,29 @@ define amdgpu_kernel void @local_system_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6069,6 +6998,29 @@ define amdgpu_kernel void @local_system_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6200,6 +7152,26 @@ define amdgpu_kernel void @local_system_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("one-as") unordered, align 4
@@ -6329,6 +7301,26 @@ define amdgpu_kernel void @local_system_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("one-as") monotonic, align 4
@@ -6458,6 +7450,26 @@ define amdgpu_kernel void @local_system_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("one-as") acquire, align 4
@@ -6587,6 +7599,26 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("one-as") seq_cst, align 4
@@ -6694,6 +7726,22 @@ define amdgpu_kernel void @local_system_one_as_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("one-as") unordered, align 4
@@ -6800,6 +7848,22 @@ define amdgpu_kernel void @local_system_one_as_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("one-as") monotonic, align 4
@@ -6906,6 +7970,22 @@ define amdgpu_kernel void @local_system_one_as_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("one-as") release, align 4
@@ -7012,6 +8092,22 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("one-as") seq_cst, align 4
@@ -7118,6 +8214,22 @@ define amdgpu_kernel void @local_system_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") monotonic
@@ -7224,6 +8336,22 @@ define amdgpu_kernel void @local_system_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") acquire
@@ -7330,6 +8458,22 @@ define amdgpu_kernel void @local_system_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") release
@@ -7436,6 +8580,22 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") acq_rel
@@ -7542,6 +8702,22 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") seq_cst
@@ -7670,6 +8846,26 @@ define amdgpu_kernel void @local_system_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") acquire
@@ -7799,6 +8995,26 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") acq_rel
@@ -7928,6 +9144,26 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") seq_cst
@@ -8047,6 +9283,24 @@ define amdgpu_kernel void @local_system_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8166,6 +9420,24 @@ define amdgpu_kernel void @local_system_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8285,6 +9557,24 @@ define amdgpu_kernel void @local_system_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8404,6 +9694,24 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8523,6 +9831,24 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8642,6 +9968,24 @@ define amdgpu_kernel void @local_system_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8761,6 +10105,24 @@ define amdgpu_kernel void @local_system_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8880,6 +10242,24 @@ define amdgpu_kernel void @local_system_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8999,6 +10379,24 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9118,6 +10516,24 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9237,6 +10653,24 @@ define amdgpu_kernel void @local_system_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9356,6 +10790,24 @@ define amdgpu_kernel void @local_system_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9475,6 +10927,24 @@ define amdgpu_kernel void @local_system_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9594,6 +11064,24 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9713,6 +11201,24 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9854,6 +11360,28 @@ define amdgpu_kernel void @local_system_one_as_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9997,6 +11525,28 @@ define amdgpu_kernel void @local_system_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10140,6 +11690,28 @@ define amdgpu_kernel void @local_system_one_as_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10283,6 +11855,28 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10426,6 +12020,28 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10569,6 +12185,28 @@ define amdgpu_kernel void @local_system_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10712,6 +12350,28 @@ define amdgpu_kernel void @local_system_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10855,6 +12515,28 @@ define amdgpu_kernel void @local_system_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10998,6 +12680,28 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11141,6 +12845,28 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11284,6 +13010,28 @@ define amdgpu_kernel void @local_system_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11427,6 +13175,28 @@ define amdgpu_kernel void @local_system_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11570,6 +13340,28 @@ define amdgpu_kernel void @local_system_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11713,6 +13505,28 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11856,6 +13670,28 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
index 1dda86e6e09879..ac09f65c32f417 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
@@ -6,6 +6,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @local_volatile_load_0(
; GFX6-LABEL: local_volatile_load_0:
@@ -103,6 +105,34 @@ define amdgpu_kernel void @local_volatile_load_0(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_volatile_load_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_volatile_load_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(1) %out) {
entry:
%val = load volatile i32, ptr addrspace(3) %in, align 4
@@ -211,6 +241,36 @@ define amdgpu_kernel void @local_volatile_load_1(
; GFX11-CU-NEXT: s_nop 0
; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_volatile_load_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s2
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_volatile_load_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s2
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(1) %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -309,6 +369,26 @@ define amdgpu_kernel void @local_volatile_store_0(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_volatile_store_0:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_volatile_store_0:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(3) %out) {
entry:
%val = load i32, ptr addrspace(1) %in, align 4
@@ -410,6 +490,28 @@ define amdgpu_kernel void @local_volatile_store_1(
; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_volatile_store_1:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s2
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_volatile_store_1:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s2
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(1) %in, ptr addrspace(3) %out) {
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -499,6 +601,27 @@ define amdgpu_kernel void @local_volatile_workgroup_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_volatile_workgroup_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_volatile_workgroup_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic volatile i32, ptr addrspace(3) %in syncscope("workgroup") acquire, align 4
@@ -570,6 +693,22 @@ define amdgpu_kernel void @local_volatile_workgroup_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_volatile_workgroup_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_volatile_workgroup_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic volatile i32 %in, ptr addrspace(3) %out syncscope("workgroup") release, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
index 1e9da50285dfa7..d210de64b3fb95 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @local_wavefront_unordered_load(
; GFX6-LABEL: local_wavefront_unordered_load:
@@ -133,6 +135,26 @@ define amdgpu_kernel void @local_wavefront_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront") unordered, align 4
@@ -262,6 +284,26 @@ define amdgpu_kernel void @local_wavefront_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront") monotonic, align 4
@@ -391,6 +433,26 @@ define amdgpu_kernel void @local_wavefront_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront") acquire, align 4
@@ -520,6 +582,26 @@ define amdgpu_kernel void @local_wavefront_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront") seq_cst, align 4
@@ -627,6 +709,22 @@ define amdgpu_kernel void @local_wavefront_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront") unordered, align 4
@@ -733,6 +831,22 @@ define amdgpu_kernel void @local_wavefront_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 4
@@ -839,6 +953,22 @@ define amdgpu_kernel void @local_wavefront_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront") release, align 4
@@ -945,6 +1075,22 @@ define amdgpu_kernel void @local_wavefront_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront") seq_cst, align 4
@@ -1051,6 +1197,22 @@ define amdgpu_kernel void @local_wavefront_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") monotonic
@@ -1157,6 +1319,22 @@ define amdgpu_kernel void @local_wavefront_acquire_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") acquire
@@ -1263,6 +1441,22 @@ define amdgpu_kernel void @local_wavefront_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") release
@@ -1369,6 +1563,22 @@ define amdgpu_kernel void @local_wavefront_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") acq_rel
@@ -1475,6 +1685,22 @@ define amdgpu_kernel void @local_wavefront_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") seq_cst
@@ -1603,6 +1829,26 @@ define amdgpu_kernel void @local_wavefront_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") acquire
@@ -1732,6 +1978,26 @@ define amdgpu_kernel void @local_wavefront_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") acq_rel
@@ -1861,6 +2127,26 @@ define amdgpu_kernel void @local_wavefront_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") seq_cst
@@ -1980,6 +2266,24 @@ define amdgpu_kernel void @local_wavefront_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2099,6 +2403,24 @@ define amdgpu_kernel void @local_wavefront_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2218,6 +2540,24 @@ define amdgpu_kernel void @local_wavefront_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2337,6 +2677,24 @@ define amdgpu_kernel void @local_wavefront_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2456,6 +2814,24 @@ define amdgpu_kernel void @local_wavefront_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2575,6 +2951,24 @@ define amdgpu_kernel void @local_wavefront_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2694,6 +3088,24 @@ define amdgpu_kernel void @local_wavefront_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2813,6 +3225,24 @@ define amdgpu_kernel void @local_wavefront_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2932,6 +3362,24 @@ define amdgpu_kernel void @local_wavefront_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3051,6 +3499,24 @@ define amdgpu_kernel void @local_wavefront_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3170,6 +3636,24 @@ define amdgpu_kernel void @local_wavefront_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3289,6 +3773,24 @@ define amdgpu_kernel void @local_wavefront_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3408,6 +3910,24 @@ define amdgpu_kernel void @local_wavefront_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3527,6 +4047,24 @@ define amdgpu_kernel void @local_wavefront_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3646,6 +4184,24 @@ define amdgpu_kernel void @local_wavefront_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3787,6 +4343,28 @@ define amdgpu_kernel void @local_wavefront_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3930,6 +4508,28 @@ define amdgpu_kernel void @local_wavefront_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4073,6 +4673,28 @@ define amdgpu_kernel void @local_wavefront_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4216,6 +4838,28 @@ define amdgpu_kernel void @local_wavefront_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4359,6 +5003,28 @@ define amdgpu_kernel void @local_wavefront_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4502,6 +5168,28 @@ define amdgpu_kernel void @local_wavefront_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4645,6 +5333,28 @@ define amdgpu_kernel void @local_wavefront_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4788,6 +5498,28 @@ define amdgpu_kernel void @local_wavefront_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4931,6 +5663,28 @@ define amdgpu_kernel void @local_wavefront_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5074,6 +5828,28 @@ define amdgpu_kernel void @local_wavefront_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5217,6 +5993,28 @@ define amdgpu_kernel void @local_wavefront_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5360,6 +6158,28 @@ define amdgpu_kernel void @local_wavefront_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5503,6 +6323,28 @@ define amdgpu_kernel void @local_wavefront_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5646,6 +6488,28 @@ define amdgpu_kernel void @local_wavefront_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5789,6 +6653,28 @@ define amdgpu_kernel void @local_wavefront_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5920,6 +6806,26 @@ define amdgpu_kernel void @local_wavefront_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront-one-as") unordered, align 4
@@ -6049,6 +6955,26 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront-one-as") monotonic, align 4
@@ -6178,6 +7104,26 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront-one-as") acquire, align 4
@@ -6307,6 +7253,26 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront-one-as") seq_cst, align 4
@@ -6414,6 +7380,22 @@ define amdgpu_kernel void @local_wavefront_one_as_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront-one-as") unordered, align 4
@@ -6520,6 +7502,22 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront-one-as") monotonic, align 4
@@ -6626,6 +7624,22 @@ define amdgpu_kernel void @local_wavefront_one_as_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront-one-as") release, align 4
@@ -6732,6 +7746,22 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront-one-as") seq_cst, align 4
@@ -6838,6 +7868,22 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") monotonic
@@ -6944,6 +7990,22 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -7050,6 +8112,22 @@ define amdgpu_kernel void @local_wavefront_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") release
@@ -7156,6 +8234,22 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -7262,6 +8356,22 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -7390,6 +8500,26 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -7519,6 +8649,26 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -7648,6 +8798,26 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -7767,6 +8937,24 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7886,6 +9074,24 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8005,6 +9211,24 @@ define amdgpu_kernel void @local_wavefront_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8124,6 +9348,24 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8243,6 +9485,24 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8362,6 +9622,24 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8481,6 +9759,24 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8600,6 +9896,24 @@ define amdgpu_kernel void @local_wavefront_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8719,6 +10033,24 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8838,6 +10170,24 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8957,6 +10307,24 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9076,6 +10444,24 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9195,6 +10581,24 @@ define amdgpu_kernel void @local_wavefront_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9314,6 +10718,24 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9433,6 +10855,24 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9574,6 +11014,28 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_monotonic_ret_cmpxch
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9717,6 +11179,28 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9860,6 +11344,28 @@ define amdgpu_kernel void @local_wavefront_one_as_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10003,6 +11509,28 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10146,6 +11674,28 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10289,6 +11839,28 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10432,6 +12004,28 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10575,6 +12169,28 @@ define amdgpu_kernel void @local_wavefront_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10718,6 +12334,28 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10861,6 +12499,28 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11004,6 +12664,28 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11147,6 +12829,28 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11290,6 +12994,28 @@ define amdgpu_kernel void @local_wavefront_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11433,6 +13159,28 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11576,6 +13324,28 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
index 652652039b6376..6f288bcf98af0e 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
@@ -10,6 +10,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
define amdgpu_kernel void @local_workgroup_unordered_load(
; GFX6-LABEL: local_workgroup_unordered_load:
@@ -133,6 +135,26 @@ define amdgpu_kernel void @local_workgroup_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") unordered, align 4
@@ -262,6 +284,26 @@ define amdgpu_kernel void @local_workgroup_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") monotonic, align 4
@@ -395,6 +437,27 @@ define amdgpu_kernel void @local_workgroup_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") acquire, align 4
@@ -528,6 +591,27 @@ define amdgpu_kernel void @local_workgroup_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") seq_cst, align 4
@@ -635,6 +719,22 @@ define amdgpu_kernel void @local_workgroup_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") unordered, align 4
@@ -741,6 +841,22 @@ define amdgpu_kernel void @local_workgroup_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") monotonic, align 4
@@ -847,6 +963,22 @@ define amdgpu_kernel void @local_workgroup_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") release, align 4
@@ -953,6 +1085,22 @@ define amdgpu_kernel void @local_workgroup_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") seq_cst, align 4
@@ -1059,6 +1207,22 @@ define amdgpu_kernel void @local_workgroup_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") monotonic
@@ -1178,6 +1342,25 @@ define amdgpu_kernel void @local_workgroup_acquire_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acquire
@@ -1284,6 +1467,22 @@ define amdgpu_kernel void @local_workgroup_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") release
@@ -1403,6 +1602,25 @@ define amdgpu_kernel void @local_workgroup_acq_rel_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acq_rel
@@ -1522,6 +1740,25 @@ define amdgpu_kernel void @local_workgroup_seq_cst_atomicrmw(
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") seq_cst
@@ -1654,6 +1891,27 @@ define amdgpu_kernel void @local_workgroup_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acquire
@@ -1787,6 +2045,27 @@ define amdgpu_kernel void @local_workgroup_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acq_rel
@@ -1920,6 +2199,27 @@ define amdgpu_kernel void @local_workgroup_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") seq_cst
@@ -2039,6 +2339,24 @@ define amdgpu_kernel void @local_workgroup_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2171,6 +2489,27 @@ define amdgpu_kernel void @local_workgroup_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2290,6 +2629,24 @@ define amdgpu_kernel void @local_workgroup_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2422,6 +2779,27 @@ define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2554,6 +2932,27 @@ define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2686,6 +3085,27 @@ define amdgpu_kernel void @local_workgroup_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2818,6 +3238,27 @@ define amdgpu_kernel void @local_workgroup_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2950,6 +3391,27 @@ define amdgpu_kernel void @local_workgroup_release_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3082,6 +3544,27 @@ define amdgpu_kernel void @local_workgroup_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3214,6 +3697,27 @@ define amdgpu_kernel void @local_workgroup_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3346,6 +3850,27 @@ define amdgpu_kernel void @local_workgroup_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3478,6 +4003,27 @@ define amdgpu_kernel void @local_workgroup_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3610,6 +4156,27 @@ define amdgpu_kernel void @local_workgroup_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3742,6 +4309,27 @@ define amdgpu_kernel void @local_workgroup_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3874,6 +4462,27 @@ define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4015,6 +4624,28 @@ define amdgpu_kernel void @local_workgroup_monotonic_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4162,6 +4793,29 @@ define amdgpu_kernel void @local_workgroup_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4305,6 +4959,28 @@ define amdgpu_kernel void @local_workgroup_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4452,6 +5128,29 @@ define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4599,6 +5298,29 @@ define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4746,6 +5468,29 @@ define amdgpu_kernel void @local_workgroup_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4893,6 +5638,29 @@ define amdgpu_kernel void @local_workgroup_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5040,6 +5808,29 @@ define amdgpu_kernel void @local_workgroup_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5187,6 +5978,29 @@ define amdgpu_kernel void @local_workgroup_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5334,6 +6148,29 @@ define amdgpu_kernel void @local_workgroup_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5481,6 +6318,29 @@ define amdgpu_kernel void @local_workgroup_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5628,6 +6488,29 @@ define amdgpu_kernel void @local_workgroup_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5775,6 +6658,29 @@ define amdgpu_kernel void @local_workgroup_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5922,6 +6828,29 @@ define amdgpu_kernel void @local_workgroup_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6069,6 +6998,29 @@ define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6200,6 +7152,26 @@ define amdgpu_kernel void @local_workgroup_one_as_unordered_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_unordered_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_unordered_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") unordered, align 4
@@ -6329,6 +7301,26 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") monotonic, align 4
@@ -6458,6 +7450,26 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") acquire, align 4
@@ -6587,6 +7599,26 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_load(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v1, v0
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v1, v0
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v1, v0
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %in, ptr addrspace(3) %out) {
entry:
%val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") seq_cst, align 4
@@ -6694,6 +7726,22 @@ define amdgpu_kernel void @local_workgroup_one_as_unordered_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_unordered_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_unordered_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") unordered, align 4
@@ -6800,6 +7848,22 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") monotonic, align 4
@@ -6906,6 +7970,22 @@ define amdgpu_kernel void @local_workgroup_one_as_release_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_release_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_release_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") release, align 4
@@ -7012,6 +8092,22 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_store(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_store:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_store:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
i32 %in, ptr addrspace(3) %out) {
entry:
store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") seq_cst, align 4
@@ -7118,6 +8214,22 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") monotonic
@@ -7224,6 +8336,22 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -7330,6 +8458,22 @@ define amdgpu_kernel void @local_workgroup_one_as_release_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_release_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_release_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") release
@@ -7436,6 +8580,22 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -7542,6 +8702,22 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_atomicrmw(
; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -7670,6 +8846,26 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -7799,6 +8995,26 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -7928,6 +9144,26 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_ret_atomicrmw(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v1
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in) {
entry:
%val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -8047,6 +9283,24 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8166,6 +9420,24 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8285,6 +9557,24 @@ define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8404,6 +9694,24 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8523,6 +9831,24 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8642,6 +9968,24 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8761,6 +10105,24 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8880,6 +10242,24 @@ define amdgpu_kernel void @local_workgroup_one_as_release_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8999,6 +10379,24 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9118,6 +10516,24 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9237,6 +10653,24 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9356,6 +10790,24 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9475,6 +10927,24 @@ define amdgpu_kernel void @local_workgroup_one_as_release_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9594,6 +11064,24 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9713,6 +11201,24 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9854,6 +11360,28 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_ret_cmpxch
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9997,6 +11525,28 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10140,6 +11690,28 @@ define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10283,6 +11855,28 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10426,6 +12020,28 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10569,6 +12185,28 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10712,6 +12350,28 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10855,6 +12515,28 @@ define amdgpu_kernel void @local_workgroup_one_as_release_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10998,6 +12680,28 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11141,6 +12845,28 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11284,6 +13010,28 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11427,6 +13175,28 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11570,6 +13340,28 @@ define amdgpu_kernel void @local_workgroup_one_as_release_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11713,6 +13505,28 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11856,6 +13670,28 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: ds_store_b32 v0, v1
; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: ds_store_b32 v0, v1
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: ds_store_b32 v0, v1
+; GFX12-CU-NEXT: s_endpgm
ptr addrspace(3) %out, i32 %in, i32 %old) {
entry:
%gep = getelementptr i32, ptr addrspace(3) %out, i32 4
>From 8540ea529ffef9fd87cff4b7e3078c01c6bd1320 Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Fri, 1 Mar 2024 18:25:40 +0100
Subject: [PATCH 2/4] Add nontemporal+volatile tests
---
.../memory-legalizer-flat-nontemporal.ll | 165 ++++++++++++++
.../memory-legalizer-global-nontemporal.ll | 158 ++++++++++++++
.../memory-legalizer-local-nontemporal.ll | 179 +++++++++++++++
.../memory-legalizer-private-nontemporal.ll | 203 ++++++++++++++++++
4 files changed, 705 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index 9f0556023aa24a..35a9ab72b04362 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -684,5 +684,170 @@ entry:
ret void
}
+define amdgpu_kernel void @flat_nontemporal_volatile_load(
+; GFX7-LABEL: flat_nontemporal_volatile_load:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: flat_store_dword v[0:1], v2
+; GFX7-NEXT: s_endpgm
+;
+; GFX10-WGP-LABEL: flat_nontemporal_volatile_load:
+; GFX10-WGP: ; %bb.0: ; %entry
+; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT: s_endpgm
+;
+; GFX10-CU-LABEL: flat_nontemporal_volatile_load:
+; GFX10-CU: ; %bb.0: ; %entry
+; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT: s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_nontemporal_volatile_load:
+; SKIP-CACHE-INV: ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
+; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT: s_endpgm
+;
+; GFX90A-NOTTGSPLIT-LABEL: flat_nontemporal_volatile_load:
+; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
+; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
+; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX90A-TGSPLIT-LABEL: flat_nontemporal_volatile_load:
+; GFX90A-TGSPLIT: ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
+; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
+; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
+; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
+; GFX90A-TGSPLIT-NEXT: s_endpgm
+;
+; GFX940-NOTTGSPLIT-LABEL: flat_nontemporal_volatile_load:
+; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
+; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX940-TGSPLIT-LABEL: flat_nontemporal_volatile_load:
+; GFX940-TGSPLIT: ; %bb.0: ; %entry
+; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
+; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
+; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_endpgm
+;
+; GFX11-WGP-LABEL: flat_nontemporal_volatile_load:
+; GFX11-WGP: ; %bb.0: ; %entry
+; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc dlc
+; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX11-WGP-NEXT: s_endpgm
+;
+; GFX11-CU-LABEL: flat_nontemporal_volatile_load:
+; GFX11-CU: ; %bb.0: ; %entry
+; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc dlc
+; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_volatile_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_volatile_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
+ ptr %in, ptr %out) {
+entry:
+ %val = load volatile i32, ptr %in, align 4, !nontemporal !0
+ store i32 %val, ptr %out
+ ret void
+}
+
!0 = !{i32 1}
declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index b2751326fa784b..672f803a12235d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -674,5 +674,163 @@ entry:
ret void
}
+define amdgpu_kernel void @global_nontemporal_volatile_load(
+; GFX6-LABEL: global_nontemporal_volatile_load:
+; GFX6: ; %bb.0: ; %entry
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: buffer_load_dword v0, off, s[4:7], 0 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: s_mov_b32 s4, s2
+; GFX6-NEXT: s_mov_b32 s5, s3
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX7-LABEL: global_nontemporal_volatile_load:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dword v[0:1], v2
+; GFX7-NEXT: s_endpgm
+;
+; GFX10-WGP-LABEL: global_nontemporal_volatile_load:
+; GFX10-WGP: ; %bb.0: ; %entry
+; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT: s_endpgm
+;
+; GFX10-CU-LABEL: global_nontemporal_volatile_load:
+; GFX10-CU: ; %bb.0: ; %entry
+; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT: s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_nontemporal_volatile_load:
+; SKIP-CACHE-INV: ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s0
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s1
+; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s2
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s3
+; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT: s_endpgm
+;
+; GFX90A-NOTTGSPLIT-LABEL: global_nontemporal_volatile_load:
+; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[0:1] glc
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX90A-TGSPLIT-LABEL: global_nontemporal_volatile_load:
+; GFX90A-TGSPLIT: ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[0:1] glc
+; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX90A-TGSPLIT-NEXT: s_endpgm
+;
+; GFX940-NOTTGSPLIT-LABEL: global_nontemporal_volatile_load:
+; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[0:1] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX940-TGSPLIT-LABEL: global_nontemporal_volatile_load:
+; GFX940-TGSPLIT: ; %bb.0: ; %entry
+; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[0:1] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_endpgm
+;
+; GFX11-WGP-LABEL: global_nontemporal_volatile_load:
+; GFX11-WGP: ; %bb.0: ; %entry
+; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc dlc
+; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX11-WGP-NEXT: s_nop 0
+; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-WGP-NEXT: s_endpgm
+;
+; GFX11-CU-LABEL: global_nontemporal_volatile_load:
+; GFX11-CU: ; %bb.0: ; %entry
+; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc dlc
+; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX11-CU-NEXT: s_nop 0
+; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_nontemporal_volatile_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_nontemporal_volatile_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
+ ptr addrspace(1) %in, ptr addrspace(1) %out) {
+entry:
+ %val = load volatile i32, ptr addrspace(1) %in, align 4, !nontemporal !0
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
!0 = !{i32 1}
declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
index 0eb2f7b1d694ec..16ede2665400ec 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -717,5 +717,184 @@ entry:
ret void
}
+define amdgpu_kernel void @local_nontemporal_volatile_load(
+; GFX6-LABEL: local_nontemporal_volatile_load:
+; GFX6: ; %bb.0: ; %entry
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX7-LABEL: local_nontemporal_volatile_load:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT: s_mov_b32 m0, -1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: ds_read_b32 v2, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: flat_store_dword v[0:1], v2
+; GFX7-NEXT: s_endpgm
+;
+; GFX10-WGP-LABEL: local_nontemporal_volatile_load:
+; GFX10-WGP: ; %bb.0: ; %entry
+; GFX10-WGP-NEXT: s_clause 0x1
+; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT: ds_read_b32 v0, v0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT: s_endpgm
+;
+; GFX10-CU-LABEL: local_nontemporal_volatile_load:
+; GFX10-CU: ; %bb.0: ; %entry
+; GFX10-CU-NEXT: s_clause 0x1
+; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT: ds_read_b32 v0, v0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT: s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_nontemporal_volatile_load:
+; SKIP-CACHE-INV: ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
+; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT: ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT: s_endpgm
+;
+; GFX90A-NOTTGSPLIT-LABEL: local_nontemporal_volatile_load:
+; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v0, v0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX90A-TGSPLIT-LABEL: local_nontemporal_volatile_load:
+; GFX90A-TGSPLIT: ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-TGSPLIT-NEXT: ds_read_b32 v0, v0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-TGSPLIT-NEXT: s_endpgm
+;
+; GFX940-NOTTGSPLIT-LABEL: local_nontemporal_volatile_load:
+; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s4
+; GFX940-NOTTGSPLIT-NEXT: ds_read_b32 v0, v0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX940-TGSPLIT-LABEL: local_nontemporal_volatile_load:
+; GFX940-TGSPLIT: ; %bb.0: ; %entry
+; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s4
+; GFX940-TGSPLIT-NEXT: ds_read_b32 v0, v0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_endpgm
+;
+; GFX11-WGP-LABEL: local_nontemporal_volatile_load:
+; GFX11-WGP: ; %bb.0: ; %entry
+; GFX11-WGP-NEXT: s_clause 0x1
+; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX11-WGP-NEXT: ds_load_b32 v0, v0
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-WGP-NEXT: s_nop 0
+; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-WGP-NEXT: s_endpgm
+;
+; GFX11-CU-LABEL: local_nontemporal_volatile_load:
+; GFX11-CU: ; %bb.0: ; %entry
+; GFX11-CU-NEXT: s_clause 0x1
+; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX11-CU-NEXT: ds_load_b32 v0, v0
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-CU-NEXT: s_nop 0
+; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_nontemporal_volatile_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_nontemporal_volatile_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
+ ptr addrspace(3) %in, ptr addrspace(1) %out) {
+entry:
+ %val = load volatile i32, ptr addrspace(3) %in, align 4, !nontemporal !0
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
!0 = !{i32 1}
declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index 069443791b3e1b..e0eff90c4f40ec 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -813,5 +813,208 @@ entry:
ret void
}
+define amdgpu_kernel void @private_nontemporal_volatile_load(
+; GFX6-LABEL: private_nontemporal_volatile_load:
+; GFX6: ; %bb.0: ; %entry
+; GFX6-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX6-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX6-NEXT: s_add_u32 s8, s8, s7
+; GFX6-NEXT: s_addc_u32 s9, s9, 0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX7-LABEL: private_nontemporal_volatile_load:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT: s_add_u32 s8, s8, s7
+; GFX7-NEXT: s_addc_u32 s9, s9, 0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: buffer_load_dword v2, v0, s[8:11], 0 offen glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_store_dword v[0:1], v2
+; GFX7-NEXT: s_endpgm
+;
+; GFX10-WGP-LABEL: private_nontemporal_volatile_load:
+; GFX10-WGP: ; %bb.0: ; %entry
+; GFX10-WGP-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX10-WGP-NEXT: s_clause 0x1
+; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT: s_add_u32 s8, s8, s7
+; GFX10-WGP-NEXT: s_addc_u32 s9, s9, 0
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc dlc
+; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT: s_endpgm
+;
+; GFX10-CU-LABEL: private_nontemporal_volatile_load:
+; GFX10-CU: ; %bb.0: ; %entry
+; GFX10-CU-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX10-CU-NEXT: s_clause 0x1
+; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT: s_add_u32 s8, s8, s7
+; GFX10-CU-NEXT: s_addc_u32 s9, s9, 0
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc dlc
+; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT: s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: private_nontemporal_volatile_load:
+; SKIP-CACHE-INV: ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT: s_getpc_b64 s[4:5]
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s0
+; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
+; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s4, s3
+; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s5, 0
+; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen glc
+; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT: s_endpgm
+;
+; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_volatile_load:
+; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: s_add_u32 s8, s8, s7
+; GFX90A-NOTTGSPLIT-NEXT: s_addc_u32 s9, s9, 0
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NOTTGSPLIT-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX90A-TGSPLIT-LABEL: private_nontemporal_volatile_load:
+; GFX90A-TGSPLIT: ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90A-TGSPLIT-NEXT: s_add_u32 s8, s8, s7
+; GFX90A-TGSPLIT-NEXT: s_addc_u32 s9, s9, 0
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-TGSPLIT-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc
+; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-TGSPLIT-NEXT: s_endpgm
+;
+; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_volatile_load:
+; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: scratch_load_dword v0, off, s4 sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX940-TGSPLIT-LABEL: private_nontemporal_volatile_load:
+; GFX940-TGSPLIT: ; %bb.0: ; %entry
+; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: scratch_load_dword v0, off, s4 sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-TGSPLIT-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_endpgm
+;
+; GFX11-WGP-LABEL: private_nontemporal_volatile_load:
+; GFX11-WGP: ; %bb.0: ; %entry
+; GFX11-WGP-NEXT: s_clause 0x1
+; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX11-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: scratch_load_b32 v0, off, s2 glc dlc
+; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX11-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-WGP-NEXT: s_nop 0
+; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-WGP-NEXT: s_endpgm
+;
+; GFX11-CU-LABEL: private_nontemporal_volatile_load:
+; GFX11-CU: ; %bb.0: ; %entry
+; GFX11-CU-NEXT: s_clause 0x1
+; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX11-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: scratch_load_b32 v0, off, s2 glc dlc
+; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-CU-NEXT: s_nop 0
+; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: private_nontemporal_volatile_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: private_nontemporal_volatile_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
+ ptr addrspace(5) %in, ptr addrspace(1) %out) {
+entry:
+ %val = load volatile i32, ptr addrspace(5) %in, align 4, !nontemporal !0
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
!0 = !{i32 1}
declare i32 @llvm.amdgcn.workitem.id.x()
>From c5f3f7c34f00e10d1b4031403a2504d8f8ae75fb Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Fri, 1 Mar 2024 18:29:18 +0100
Subject: [PATCH 3/4] Propely set nontemporal when volatile is also present
Iterator MI can advance in insertWait() but we need original
instruction to set temporal hint. Just move it before handling
volatile.
---
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 10 +++++-----
.../AMDGPU/memory-legalizer-flat-nontemporal.ll | 4 ++--
.../AMDGPU/memory-legalizer-global-nontemporal.ll | 4 ++--
.../AMDGPU/memory-legalizer-private-nontemporal.ll | 4 ++--
4 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 4069a368f68719..73c23f0f987c3d 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -2392,6 +2392,11 @@ bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal(
bool Changed = false;
+ if (IsNonTemporal) {
+ // Set non-temporal hint for all cache levels.
+ Changed |= setTH(MI, AMDGPU::CPol::TH_NT);
+ }
+
if (IsVolatile) {
Changed |= setScope(MI, AMDGPU::CPol::SCOPE_SYS);
@@ -2407,11 +2412,6 @@ bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal(
Position::AFTER);
}
- if (IsNonTemporal) {
- // Set non-temporal hint for all cache levels.
- Changed |= setTH(MI, AMDGPU::CPol::TH_NT);
- }
-
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index 35a9ab72b04362..f6fac11d634c5b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -824,7 +824,7 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-WGP-NEXT: s_wait_dscnt 0x0
@@ -836,7 +836,7 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-CU-NEXT: s_wait_dscnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index 672f803a12235d..c95ff2c691a3cb 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -807,7 +807,7 @@ define amdgpu_kernel void @global_nontemporal_volatile_load(
; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
; GFX12-WGP-NEXT: s_nop 0
@@ -819,7 +819,7 @@ define amdgpu_kernel void @global_nontemporal_volatile_load(
; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
; GFX12-CU-NEXT: s_nop 0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index e0eff90c4f40ec..30296a9c3d8963 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -989,7 +989,7 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
; GFX12-WGP-NEXT: v_mov_b32_e32 v1, 0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-WGP-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-WGP-NEXT: s_nop 0
@@ -1003,7 +1003,7 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
; GFX12-CU-NEXT: v_mov_b32_e32 v1, 0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-CU-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-CU-NEXT: s_nop 0
>From b5b25657650f6800cd6e59eaf6d4b8ba6458d2de Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Mon, 4 Mar 2024 11:03:10 +0100
Subject: [PATCH 4/4] [AMDGPU] Handle amdgpu.last.use metadata
Convert !amdgpu.last.use metadata into Machine Memory Operand for last use
and handle it in SIMemoryLegalizer similary to nontemporal and volatile.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 +-
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 56 ++++++++++++++-
.../AMDGPU/memory-legalizer-flat-lastuse.ll | 66 +++++++++++++++++
.../AMDGPU/memory-legalizer-global-lastuse.ll | 65 +++++++++++++++++
.../memory-legalizer-private-lastuse.ll | 70 +++++++++++++++++++
5 files changed, 259 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0df0b5cdf0f392..72c79cc58f8ea9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16181,9 +16181,12 @@ bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
MachineMemOperand::Flags
SITargetLowering::getTargetMMOFlags(const Instruction &I) const {
// Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
+ MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
if (I.getMetadata("amdgpu.noclobber"))
- return MONoClobber;
- return MachineMemOperand::MONone;
+ Flags |= MONoClobber;
+ if (I.getMetadata("amdgpu.last.use"))
+ Flags |= MOLastUse;
+ return Flags;
}
bool SITargetLowering::checkForPhysRegDependency(
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 73c23f0f987c3d..ff3e6bc9f8b08f 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -98,6 +98,7 @@ class SIMemOpInfo final {
bool IsCrossAddressSpaceOrdering = false;
bool IsVolatile = false;
bool IsNonTemporal = false;
+ bool IsLastUse = false;
SIMemOpInfo(AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent,
SIAtomicScope Scope = SIAtomicScope::SYSTEM,
@@ -107,13 +108,15 @@ class SIMemOpInfo final {
AtomicOrdering FailureOrdering =
AtomicOrdering::SequentiallyConsistent,
bool IsVolatile = false,
- bool IsNonTemporal = false)
+ bool IsNonTemporal = false,
+ bool IsLastUse = false)
: Ordering(Ordering), FailureOrdering(FailureOrdering),
Scope(Scope), OrderingAddrSpace(OrderingAddrSpace),
InstrAddrSpace(InstrAddrSpace),
IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
IsVolatile(IsVolatile),
- IsNonTemporal(IsNonTemporal) {
+ IsNonTemporal(IsNonTemporal),
+ IsLastUse(IsLastUse) {
if (Ordering == AtomicOrdering::NotAtomic) {
assert(Scope == SIAtomicScope::NONE &&
@@ -201,6 +204,12 @@ class SIMemOpInfo final {
return IsNonTemporal;
}
+ /// \returns True if memory access of the machine instruction used to
+ /// create this SIMemOpInfo is last use, false otherwise.
+ bool isLastUse() const {
+ return IsLastUse;
+ }
+
/// \returns True if ordering constraint of the machine instruction used to
/// create this SIMemOpInfo is unordered or higher, false otherwise.
bool isAtomic() const {
@@ -316,6 +325,12 @@ class SICacheControl {
return false;
};
+ /// Update \p MI memory instruction to indicate it is a last use. Return true
+ /// iff the instruction was modified.
+ virtual bool enableLastUse(MachineInstr &MI, bool IsLastUse) const {
+ return false;
+ }
+
/// Inserts any necessary instructions at position \p Pos relative
/// to instruction \p MI to ensure memory instructions before \p Pos of kind
/// \p Op associated with address spaces \p AddrSpace have completed. Used
@@ -592,6 +607,10 @@ class SIGfx12CacheControl : public SIGfx11CacheControl {
// MI. \returns Returns true if \p MI is modified, false otherwise.
bool setScope(const MachineBasicBlock::iterator MI,
AMDGPU::CPol::CPol Value) const;
+ // Checks if CPol operand is present in instruction \p MI and if current Scope
+ // policy is same as \p Value.
+ bool isScope(const MachineBasicBlock::iterator MI,
+ AMDGPU::CPol::CPol Value) const;
// Stores with system scope (SCOPE_SYS) need to wait for:
// - loads or atomics(returning) - wait for {LOAD|SAMPLE|BVH|KM}CNT==0
@@ -618,6 +637,9 @@ class SIGfx12CacheControl : public SIGfx11CacheControl {
bool IsNonTemporal) const override;
bool expandSystemScopeStore(MachineBasicBlock::iterator &MI) const override;
+
+ bool enableLastUse(MachineInstr &MI,
+ bool IsLastUse) const override;
};
class SIMemoryLegalizer final : public MachineFunctionPass {
@@ -745,12 +767,14 @@ std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE;
bool IsNonTemporal = true;
bool IsVolatile = false;
+ bool IsLastUse = false;
// Validator should check whether or not MMOs cover the entire set of
// locations accessed by the memory instruction.
for (const auto &MMO : MI->memoperands()) {
IsNonTemporal &= MMO->isNonTemporal();
IsVolatile |= MMO->isVolatile();
+ IsLastUse |= MMO->getFlags() & MOLastUse;
InstrAddrSpace |=
toSIAtomicAddrSpace(MMO->getPointerInfo().getAddrSpace());
AtomicOrdering OpOrdering = MMO->getSuccessOrdering();
@@ -792,7 +816,7 @@ std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
}
return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, InstrAddrSpace,
IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile,
- IsNonTemporal);
+ IsNonTemporal, IsLastUse);
}
std::optional<SIMemOpInfo>
@@ -2209,6 +2233,15 @@ bool SIGfx12CacheControl::setScope(const MachineBasicBlock::iterator MI,
return false;
}
+bool SIGfx12CacheControl::isScope(const MachineBasicBlock::iterator MI,
+ AMDGPU::CPol::CPol Value) const {
+ MachineOperand *CPol = TII->getNamedOperand(*MI, OpName::cpol);
+ if (!CPol)
+ return false;
+
+ return (CPol->getImm() & AMDGPU::CPol::SCOPE) == Value;
+}
+
bool SIGfx12CacheControl::insertWaitsBeforeSystemScopeStore(
const MachineBasicBlock::iterator MI) const {
// TODO: implement flag for frontend to give us a hint not to insert waits.
@@ -2415,6 +2448,16 @@ bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal(
return Changed;
}
+bool SIGfx12CacheControl::enableLastUse(MachineInstr &MI,
+ bool IsLastUse) const {
+ assert(MI.mayLoad() && !MI.mayStore());
+
+ if (IsLastUse && !isScope(MI, AMDGPU::CPol::SCOPE_SYS))
+ return setTH(MI, AMDGPU::CPol::TH_LU);;
+
+ return false;
+}
+
bool SIGfx12CacheControl::expandSystemScopeStore(
MachineBasicBlock::iterator &MI) const {
MachineOperand *CPol = TII->getNamedOperand(*MI, OpName::cpol);
@@ -2471,12 +2514,19 @@ bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
return Changed;
}
+ // enableVolatileAndOrNonTemporal can insert instructions and advance iterator
+ // MI and we need original instruction for enabling last use.
+ MachineInstr &Inst = *MI;
+
// Atomic instructions already bypass caches to the scope specified by the
// SyncScope operand. Only non-atomic volatile and nontemporal instructions
// need additional treatment.
Changed |= CC->enableVolatileAndOrNonTemporal(MI, MOI.getInstrAddrSpace(),
SIMemOp::LOAD, MOI.isVolatile(),
MOI.isNonTemporal());
+
+ Changed |= CC->enableLastUse(Inst, MOI.isLastUse());
+
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
new file mode 100644
index 00000000000000..e3ce16fe741f7b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s
+
+define amdgpu_kernel void @flat_last_use_load_0(ptr %in, ptr %out) {
+; GFX12-LABEL: flat_last_use_load_0:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_LU
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load i32, ptr %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr %out
+ ret void
+}
+
+define amdgpu_kernel void @flat_last_use_load_1(ptr %in, ptr %out) {
+; GFX12-LABEL: flat_last_use_load_1:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX12-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_LU
+; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %val.gep = getelementptr inbounds i32, ptr %in, i32 %tid
+ %val = load i32, ptr %val.gep, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr %out
+ ret void
+}
+
+define amdgpu_kernel void @flat_last_use_volatile_load(ptr %in, ptr %out) {
+; GFX12-LABEL: flat_last_use_volatile_load:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load volatile i32, ptr %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr %out
+ ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-CU: {{.*}}
+; GFX12-WGP: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
new file mode 100644
index 00000000000000..56b1a366708127
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s
+
+define amdgpu_kernel void @global_last_use_load_0(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: global_last_use_load_0:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load i32, ptr addrspace(1) %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @global_last_use_load_1(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: global_last_use_load_1:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] th:TH_LOAD_LU
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %val.gep = getelementptr inbounds i32, ptr addrspace(1) %in, i32 %tid
+ %val = load i32, ptr addrspace(1) %val.gep, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @global_last_use_volatile_load(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: global_last_use_volatile_load:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load volatile i32, ptr addrspace(1) %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-CU: {{.*}}
+; GFX12-WGP: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
new file mode 100644
index 00000000000000..f54e7819e1b56b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s
+
+define amdgpu_kernel void @private_nontemporal_load_0(ptr addrspace(5) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: private_nontemporal_load_0:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_LU
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load i32, ptr addrspace(5) %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @private_nontemporal_load_1(ptr addrspace(5) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: private_nontemporal_load_1:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: scratch_load_b32 v0, v0, s2 th:TH_LOAD_LU
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %val.gep = getelementptr inbounds i32, ptr addrspace(5) %in, i32 %tid
+ %val = load i32, ptr addrspace(5) %val.gep, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @private_nontemporal_volatile_load(ptr addrspace(5) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: private_nontemporal_volatile_load:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load volatile i32, ptr addrspace(5) %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-CU: {{.*}}
+; GFX12-WGP: {{.*}}
More information about the llvm-commits
mailing list