[llvm] [NFC][AMDGPU] Pre-commit tests for workgroup to wavefront demotion (PR #224064)
Zach Goldthorpe via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 07:53:11 PDT 2026
https://github.com/zGoldthorpe updated https://github.com/llvm/llvm-project/pull/224064
>From 2015f96ac90be426f595db5f4aed3376b7913915 Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Wed, 16 Sep 2026 09:21:48 -0500
Subject: [PATCH 1/3] [NFC][AMDGPU] Pre-commit tests for workgroup to wavefront
demotion
---
...zer-single-wave-workgroup-lds-dma-async.ll | 49 +
...legalizer-single-wave-workgroup-lds-dma.ll | 181 +
...-legalizer-single-wave-workgroup-memops.ll | 3158 +++++++++++++++++
3 files changed, 3388 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
new file mode 100644
index 0000000000000..13efb8daae0a3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu12.50 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX1250
+
+declare void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) nocapture, ptr addrspace(3) nocapture, i32, i32)
+
+define amdgpu_kernel void @lds_wg_fence_release_single32(ptr addrspace(3) %lds) #0 {
+ ; GFX1250-LABEL: name: lds_wg_fence_release_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 1, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store i32 1, ptr addrspace(3) %lds, align 4
+ fence syncscope("workgroup") release
+ %v = load i32, ptr addrspace(3) %lds, align 4
+ store i32 %v, ptr addrspace(3) %lds, align 4
+ ret void
+}
+
+define amdgpu_kernel void @lds_async_dma_wg_fence_release_single32(ptr addrspace(1) %g, ptr addrspace(3) %lds) #0 {
+ ; GFX1250-LABEL: name: lds_async_dma_wg_fence_release_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.g.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32_SADDR killed $vgpr1, killed $sgpr0_sgpr1, killed $vgpr0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32) from %ir.g.load, align 1, addrspace 1), (store (s32) into %ir.lds.load, align 1, addrspace 3)
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %g, ptr addrspace(3) %lds, i32 0, i32 0)
+ fence syncscope("workgroup") release
+ %v = load i32, ptr addrspace(3) %lds, align 4
+ store i32 %v, ptr addrspace(3) %lds, align 4
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="32,32" }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
new file mode 100644
index 0000000000000..2cfe2bb0ad0bd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.00 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX9
+; RUN: llc -mtriple=amdgpu9.42 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX942
+; RUN: llc -mtriple=amdgpu10.10 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W64
+
+declare void @llvm.amdgcn.raw.ptr.buffer.load.lds(ptr addrspace(8), ptr addrspace(3) nocapture, i32, i32, i32, i32, i32)
+
+define amdgpu_kernel void @lds_wg_fence_release_single32(ptr addrspace(3) %lds) #0 {
+ ; GFX9-LABEL: name: lds_wg_fence_release_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_fence_release_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_fence_release_single32
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX10-NEXT: S_ENDPGM 0
+ store i32 1, ptr addrspace(3) %lds, align 4
+ fence syncscope("workgroup") release
+ %v = load i32, ptr addrspace(3) %lds, align 4
+ store i32 %v, ptr addrspace(3) %lds, align 4
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_fence_release_single64(ptr addrspace(3) %lds) #1 {
+ ; GFX9-LABEL: name: lds_wg_fence_release_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_fence_release_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_fence_release_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
+ ; GFX10-NEXT: S_ENDPGM 0
+ store i32 1, ptr addrspace(3) %lds, align 4
+ fence syncscope("workgroup") release
+ %v = load i32, ptr addrspace(3) %lds, align 4
+ store i32 %v, ptr addrspace(3) %lds, align 4
+ ret void
+}
+
+define amdgpu_kernel void @lds_dma_wg_fence_release_single32(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds) #0 {
+ ; GFX9-LABEL: name: lds_dma_wg_fence_release_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr6 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 52, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.rsrc.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: $m0 = S_MOV_B32 killed $sgpr6
+ ; GFX9-NEXT: BUFFER_LOAD_DWORD_LDS_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec, implicit $m0 :: (dereferenceable load (s32) from %ir.rsrc.load, align 1, addrspace 8), (dereferenceable store (s2048) into %ir.lds.load, align 1, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_dma_wg_fence_release_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr6 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 52, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.rsrc.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: $m0 = S_MOV_B32 killed $sgpr6
+ ; GFX942-NEXT: BUFFER_LOAD_DWORD_LDS_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec, implicit $m0 :: (dereferenceable load (s32) from %ir.rsrc.load, align 1, addrspace 8), (dereferenceable store (s2048) into %ir.lds.load, align 1, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W32-LABEL: name: lds_dma_wg_fence_release_single32
+ ; GFX10-W32: bb.0 (%ir-block.0):
+ ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W32-NEXT: renamable $sgpr6 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 52, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.rsrc.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: $m0 = S_MOV_B32 killed $sgpr6
+ ; GFX10-W32-NEXT: BUFFER_LOAD_DWORD_LDS_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec, implicit $m0 :: (dereferenceable load (s32) from %ir.rsrc.load, align 1, addrspace 8), (dereferenceable store (s1024) into %ir.lds.load, align 1, addrspace 3)
+ ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W64-LABEL: name: lds_dma_wg_fence_release_single32
+ ; GFX10-W64: bb.0 (%ir-block.0):
+ ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W64-NEXT: renamable $sgpr6 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 52, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.rsrc.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: $m0 = S_MOV_B32 killed $sgpr6
+ ; GFX10-W64-NEXT: BUFFER_LOAD_DWORD_LDS_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec, implicit $m0 :: (dereferenceable load (s32) from %ir.rsrc.load, align 1, addrspace 8), (dereferenceable store (s2048) into %ir.lds.load, align 1, addrspace 3)
+ ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W64-NEXT: S_ENDPGM 0
+ call void @llvm.amdgcn.raw.ptr.buffer.load.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
+ fence syncscope("workgroup") release
+ %v = load i32, ptr addrspace(3) %lds, align 4
+ store i32 %v, ptr addrspace(3) %lds, align 4
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="32,32" }
+attributes #1 = { "amdgpu-flat-work-group-size"="64,64" }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
new file mode 100644
index 0000000000000..ab89739b75bd4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
@@ -0,0 +1,3158 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.00 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX9
+; RUN: llc -mtriple=amdgpu9.42 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX942
+; RUN: llc -mtriple=amdgpu10.10 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W64
+; RUN: llc -mtriple=amdgpu12.00 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX12,GFX12-W32
+; RUN: llc -mtriple=amdgpu12.00 -mattr=+wavefrontsize64 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX12,GFX12-W64
+; RUN: llc -mtriple=amdgpu12.50 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX1250
+
+
+define amdgpu_kernel void @wg_fence_acq_rel_single32() #0 {
+ ; GFX9-LABEL: name: wg_fence_acq_rel_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_fence_acq_rel_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_fence_acq_rel_single32
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_fence_acq_rel_single32
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_fence_acq_rel_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ fence syncscope("workgroup") acq_rel
+ ret void
+}
+
+define amdgpu_kernel void @wg_fence_acq_rel_single64() #1 {
+ ; GFX9-LABEL: name: wg_fence_acq_rel_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_fence_acq_rel_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_fence_acq_rel_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_fence_acq_rel_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_fence_acq_rel_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ fence syncscope("workgroup") acq_rel
+ ret void
+}
+
+define amdgpu_kernel void @wg_fence_acq_rel_multi() #2 {
+ ; GFX9-LABEL: name: wg_fence_acq_rel_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_fence_acq_rel_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_fence_acq_rel_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_fence_acq_rel_multi
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_fence_acq_rel_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ fence syncscope("workgroup") acq_rel
+ ret void
+}
+
+define amdgpu_kernel void @wg_fence_acquire_single64() #1 {
+ ; GFX9-LABEL: name: wg_fence_acquire_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_fence_acquire_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_fence_acquire_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_fence_acquire_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_fence_acquire_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ fence syncscope("workgroup") acquire
+ ret void
+}
+
+define amdgpu_kernel void @wg_fence_release_single64() #1 {
+ ; GFX9-LABEL: name: wg_fence_release_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_fence_release_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_fence_release_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_fence_release_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_fence_release_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ fence syncscope("workgroup") release
+ ret void
+}
+
+define amdgpu_kernel void @wg_fence_seq_cst_single64() #1 {
+ ; GFX9-LABEL: name: wg_fence_seq_cst_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_fence_seq_cst_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_fence_seq_cst_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_fence_seq_cst_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_fence_seq_cst_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ fence syncscope("workgroup") seq_cst
+ ret void
+}
+
+define amdgpu_kernel void @wg_ld_seq_cst_single32(ptr addrspace(1) %p) #0 {
+ ; GFX9-LABEL: name: wg_ld_seq_cst_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_ld_seq_cst_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_ld_seq_cst_single32
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_ld_seq_cst_single32
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_ld_seq_cst_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_ld_seq_cst_single64(ptr addrspace(1) %p) #1 {
+ ; GFX9-LABEL: name: wg_ld_seq_cst_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_ld_seq_cst_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_ld_seq_cst_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_ld_seq_cst_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_ld_seq_cst_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_ld_seq_cst_multi(ptr addrspace(1) %p) #2 {
+ ; GFX9-LABEL: name: wg_ld_seq_cst_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_ld_seq_cst_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_ld_seq_cst_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_ld_seq_cst_multi
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_ld_seq_cst_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_ld_acquire_single64(ptr addrspace(1) %p) #1 {
+ ; GFX9-LABEL: name: wg_ld_acquire_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_ld_acquire_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_ld_acquire_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_ld_acquire_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_ld_acquire_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") acquire, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_ld_monotonic_single64(ptr addrspace(1) %p) #1 {
+ ; GFX9-LABEL: name: wg_ld_monotonic_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_ld_monotonic_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_ld_monotonic_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_ld_monotonic_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_ld_monotonic_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_st_seq_cst_single32(ptr addrspace(1) %p, i32 %x) #0 {
+ ; GFX9-LABEL: name: wg_st_seq_cst_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_st_seq_cst_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_st_seq_cst_single32
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_st_seq_cst_single32
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_st_seq_cst_single32
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_st_seq_cst_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_st_seq_cst_single64(ptr addrspace(1) %p, i32 %x) #1 {
+ ; GFX9-LABEL: name: wg_st_seq_cst_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_st_seq_cst_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_st_seq_cst_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_st_seq_cst_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_st_seq_cst_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_st_seq_cst_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_st_seq_cst_multi(ptr addrspace(1) %p, i32 %x) #2 {
+ ; GFX9-LABEL: name: wg_st_seq_cst_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_st_seq_cst_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_st_seq_cst_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_st_seq_cst_multi
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_st_seq_cst_multi
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_st_seq_cst_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_st_release_single64(ptr addrspace(1) %p, i32 %x) #1 {
+ ; GFX9-LABEL: name: wg_st_release_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_st_release_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_st_release_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_st_release_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_st_release_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_st_release_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") release, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) %p) #0 {
+ ; GFX9-LABEL: name: wg_rmw_add_seq_cst_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.1 (%ir-block.11):
+ ; GFX9-NEXT: successors: %bb.2(0x80000000)
+ ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.2 (%ir-block.16):
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_rmw_add_seq_cst_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.1 (%ir-block.11):
+ ; GFX942-NEXT: successors: %bb.2(0x80000000)
+ ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.2 (%ir-block.16):
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_single32
+ ; GFX10-W32: bb.0 (%ir-block.0):
+ ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
+ ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX10-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_single32
+ ; GFX10-W64: bb.0 (%ir-block.0):
+ ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX10-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_single32
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_single32
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1 (%ir-block.7):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2 (%ir-block.11):
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
+ ret void
+}
+
+define amdgpu_kernel void @wg_rmw_add_seq_cst_single64(ptr addrspace(1) %p) #1 {
+ ; GFX9-LABEL: name: wg_rmw_add_seq_cst_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.1 (%ir-block.11):
+ ; GFX9-NEXT: successors: %bb.2(0x80000000)
+ ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.2 (%ir-block.16):
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_rmw_add_seq_cst_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.1 (%ir-block.11):
+ ; GFX942-NEXT: successors: %bb.2(0x80000000)
+ ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.2 (%ir-block.16):
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_single64
+ ; GFX10-W32: bb.0 (%ir-block.0):
+ ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
+ ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX10-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_single64
+ ; GFX10-W64: bb.0 (%ir-block.0):
+ ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX10-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1 (%ir-block.7):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2 (%ir-block.11):
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
+ ret void
+}
+
+define amdgpu_kernel void @wg_rmw_add_seq_cst_multi(ptr addrspace(1) %p) #2 {
+ ; GFX9-LABEL: name: wg_rmw_add_seq_cst_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.1 (%ir-block.11):
+ ; GFX9-NEXT: successors: %bb.2(0x80000000)
+ ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.2 (%ir-block.16):
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_rmw_add_seq_cst_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.1 (%ir-block.11):
+ ; GFX942-NEXT: successors: %bb.2(0x80000000)
+ ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.2 (%ir-block.16):
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_multi
+ ; GFX10-W32: bb.0 (%ir-block.0):
+ ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
+ ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX10-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_multi
+ ; GFX10-W64: bb.0 (%ir-block.0):
+ ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX10-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_multi
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_multi
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1 (%ir-block.7):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2 (%ir-block.11):
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
+ ret void
+}
+
+define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32 %x) #1 {
+ ; GFX9-LABEL: name: wg_rmw_xchg_acq_rel_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_rmw_xchg_acq_rel_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_rmw_xchg_acq_rel_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_rmw_xchg_acq_rel_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_rmw_xchg_acq_rel_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_rmw_xchg_acq_rel_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = atomicrmw xchg ptr addrspace(1) %p, i32 %x syncscope("workgroup") acq_rel
+ ret void
+}
+
+define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) %p, i32 %cmp, i32 %new) #1 {
+ ; GFX9-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") acq_rel monotonic
+ ret void
+}
+
+define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p, i32 %cmp, i32 %new) #2 {
+ ; GFX9-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") seq_cst seq_cst
+ ret void
+}
+
+define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1) %p, i32 %cmp, i32 %new) #1 {
+ ; GFX9-LABEL: name: wg_cmpxchg_acquire_acquire_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_cmpxchg_acquire_acquire_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_cmpxchg_acquire_acquire_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_cmpxchg_acquire_acquire_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_cmpxchg_acquire_acquire_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_cmpxchg_acquire_acquire_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") acquire acquire
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) %p) #0 {
+ ; GFX9-LABEL: name: lds_wg_ld_seq_cst_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_ld_seq_cst_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_ld_seq_cst_single32
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: lds_wg_ld_seq_cst_single32
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX12-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_ld_seq_cst_single64(ptr addrspace(3) %p) #1 {
+ ; GFX9-LABEL: name: lds_wg_ld_seq_cst_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_ld_seq_cst_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_ld_seq_cst_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: lds_wg_ld_seq_cst_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX12-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_ld_seq_cst_multi(ptr addrspace(3) %p) #2 {
+ ; GFX9-LABEL: name: lds_wg_ld_seq_cst_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_ld_seq_cst_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_ld_seq_cst_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: lds_wg_ld_seq_cst_multi
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX12-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_st_release_single64(ptr addrspace(3) %p, i32 %x) #1 {
+ ; GFX9-LABEL: name: lds_wg_st_release_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_st_release_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_st_release_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: lds_wg_st_release_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: lds_wg_st_release_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: lds_wg_st_release_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store atomic i32 %x, ptr addrspace(3) %p syncscope("workgroup") release, align 4
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_rmw_add_acq_rel_single64(ptr addrspace(3) %p) #1 {
+ ; GFX9-LABEL: name: lds_wg_rmw_add_acq_rel_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.1 (%ir-block.11):
+ ; GFX9-NEXT: successors: %bb.2(0x80000000)
+ ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.2 (%ir-block.16):
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_rmw_add_acq_rel_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.1 (%ir-block.11):
+ ; GFX942-NEXT: successors: %bb.2(0x80000000)
+ ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.2 (%ir-block.16):
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W32-LABEL: name: lds_wg_rmw_add_acq_rel_single64
+ ; GFX10-W32: bb.0 (%ir-block.0):
+ ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
+ ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
+ ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W32-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
+ ; GFX10-W32-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX10-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W64-LABEL: name: lds_wg_rmw_add_acq_rel_single64
+ ; GFX10-W64: bb.0 (%ir-block.0):
+ ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
+ ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-W64-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
+ ; GFX10-W64-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX10-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: lds_wg_rmw_add_acq_rel_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
+ ; GFX12-W32-NEXT: $vgpr1, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: lds_wg_rmw_add_acq_rel_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: lds_wg_rmw_add_acq_rel_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1 (%ir-block.7):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
+ ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
+ ; GFX1250-NEXT: $vgpr1, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2 (%ir-block.11):
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = atomicrmw add ptr addrspace(3) %p, i32 3 syncscope("workgroup") acq_rel
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(3) %p, i32 %cmp, i32 %new) #1 {
+ ; GFX9-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
+ ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = cmpxchg ptr addrspace(3) %p, i32 %cmp, i32 %new syncscope("workgroup") acq_rel monotonic
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_ld_unordered_single64(ptr addrspace(3) %p) #1 {
+ ; GFX9-LABEL: name: lds_wg_ld_unordered_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_ld_unordered_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_ld_unordered_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: lds_wg_ld_unordered_single64
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: lds_wg_ld_unordered_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") unordered, align 4
+ ret void
+}
+
+define amdgpu_kernel void @lds_wg_cmpxchg_monotonic_acquire_single64(ptr addrspace(3) %p, i32 %cmp, i32 %new) #1 {
+ ; GFX9-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
+ ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = cmpxchg ptr addrspace(3) %p, i32 %cmp, i32 %new syncscope("workgroup") monotonic acquire
+ ret void
+}
+
+define amdgpu_kernel void @flat_wg_ld_acquire_single64(ptr addrspace(0) %p) #1 {
+ ; GFX9-LABEL: name: flat_wg_ld_acquire_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX9-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: flat_wg_ld_acquire_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 killed $sgpr0_sgpr1, implicit $exec, implicit $exec
+ ; GFX942-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 1, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: flat_wg_ld_acquire_single64
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX10-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 1, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: flat_wg_ld_acquire_single64
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX12-W32-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 8, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: flat_wg_ld_acquire_single64
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX12-W64-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 8, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: flat_wg_ld_acquire_single64
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(0) %p syncscope("workgroup") acquire, align 4
+ ret void
+}
+
+define amdgpu_kernel void @flat_wg_st_seq_cst_multi(ptr addrspace(0) %p, i32 %x) #2 {
+ ; GFX9-LABEL: name: flat_wg_st_seq_cst_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: flat_wg_st_seq_cst_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 killed $sgpr0_sgpr1, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX942-NEXT: S_WAITCNT_lds_direct
+ ; GFX942-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 1, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: flat_wg_st_seq_cst_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX10-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
+ ; GFX10-NEXT: S_WAITCNT_lds_direct
+ ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
+ ; GFX10-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: flat_wg_st_seq_cst_multi
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W32-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: flat_wg_st_seq_cst_multi
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX12-W64-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: flat_wg_st_seq_cst_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: FLAT_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store atomic i32 %x, ptr addrspace(0) %p syncscope("workgroup") seq_cst, align 4
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="32,32" }
+attributes #1 = { "amdgpu-flat-work-group-size"="64,64" }
+attributes #2 = { "amdgpu-flat-work-group-size"="64,256" }
>From 554da49511cc4d005c63bad820bd759befc92b22 Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Thu, 17 Sep 2026 11:50:50 -0500
Subject: [PATCH 2/3] Add more monotonic tests
---
...-legalizer-single-wave-workgroup-memops.ll | 870 +++++++++++++++++-
1 file changed, 849 insertions(+), 21 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
index ab89739b75bd4..8d465f045a993 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
@@ -608,6 +608,65 @@ define amdgpu_kernel void @wg_ld_acquire_single64(ptr addrspace(1) %p) #1 {
ret void
}
+define amdgpu_kernel void @wg_ld_monotonic_single32(ptr addrspace(1) %p) #0 {
+ ; GFX9-LABEL: name: wg_ld_monotonic_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_ld_monotonic_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_ld_monotonic_single32
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_ld_monotonic_single32
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_ld_monotonic_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
+ ret void
+}
+
define amdgpu_kernel void @wg_ld_monotonic_single64(ptr addrspace(1) %p) #1 {
; GFX9-LABEL: name: wg_ld_monotonic_single64
; GFX9: bb.0 (%ir-block.0):
@@ -667,6 +726,65 @@ define amdgpu_kernel void @wg_ld_monotonic_single64(ptr addrspace(1) %p) #1 {
ret void
}
+define amdgpu_kernel void @wg_ld_monotonic_multi(ptr addrspace(1) %p) #2 {
+ ; GFX9-LABEL: name: wg_ld_monotonic_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_ld_monotonic_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_ld_monotonic_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: wg_ld_monotonic_multi
+ ; GFX12: bb.0 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX12-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_ld_monotonic_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
+ ret void
+}
+
define amdgpu_kernel void @wg_st_seq_cst_single32(ptr addrspace(1) %p, i32 %x) #0 {
; GFX9-LABEL: name: wg_st_seq_cst_single32
; GFX9: bb.0 (%ir-block.0):
@@ -1059,6 +1177,162 @@ define amdgpu_kernel void @wg_st_release_single64(ptr addrspace(1) %p, i32 %x) #
ret void
}
+define amdgpu_kernel void @wg_st_monotonic_single32(ptr addrspace(1) %p, i32 %x) #0 {
+ ; GFX9-LABEL: name: wg_st_monotonic_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_st_monotonic_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_st_monotonic_single32
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_st_monotonic_single32
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_st_monotonic_single32
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_st_monotonic_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
+ ret void
+}
+
+define amdgpu_kernel void @wg_st_monotonic_multi(ptr addrspace(1) %p, i32 %x) #2 {
+ ; GFX9-LABEL: name: wg_st_monotonic_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_st_monotonic_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_st_monotonic_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_st_monotonic_multi
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_st_monotonic_multi
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_st_monotonic_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
+ ret void
+}
+
define amdgpu_kernel void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) %p) #0 {
; GFX9-LABEL: name: wg_rmw_add_seq_cst_single32
; GFX9: bb.0 (%ir-block.0):
@@ -1848,32 +2122,424 @@ define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32
ret void
}
-define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+define amdgpu_kernel void @wg_rmw_add_monotonic_single32(ptr addrspace(1) %p) #0 {
+ ; GFX9-LABEL: name: wg_rmw_add_monotonic_single32
; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; GFX9-NEXT: liveins: $sgpr4_sgpr5
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.1 (%ir-block.11):
+ ; GFX9-NEXT: successors: %bb.2(0x80000000)
+ ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.2 (%ir-block.16):
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_rmw_add_monotonic_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.1 (%ir-block.11):
+ ; GFX942-NEXT: successors: %bb.2(0x80000000)
+ ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.2 (%ir-block.16):
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W32-LABEL: name: wg_rmw_add_monotonic_single32
+ ; GFX10-W32: bb.0 (%ir-block.0):
+ ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
+ ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX10-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W64-LABEL: name: wg_rmw_add_monotonic_single32
+ ; GFX10-W64: bb.0 (%ir-block.0):
+ ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX10-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_rmw_add_monotonic_single32
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_rmw_add_monotonic_single32
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_rmw_add_monotonic_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1 (%ir-block.7):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2 (%ir-block.11):
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") monotonic
+ ret void
+}
+
+define amdgpu_kernel void @wg_rmw_add_monotonic_multi(ptr addrspace(1) %p) #2 {
+ ; GFX9-LABEL: name: wg_rmw_add_monotonic_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.1 (%ir-block.11):
+ ; GFX9-NEXT: successors: %bb.2(0x80000000)
+ ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: bb.2 (%ir-block.16):
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_rmw_add_monotonic_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.1 (%ir-block.11):
+ ; GFX942-NEXT: successors: %bb.2(0x80000000)
+ ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: bb.2 (%ir-block.16):
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W32-LABEL: name: wg_rmw_add_monotonic_multi
+ ; GFX10-W32: bb.0 (%ir-block.0):
+ ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
+ ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: {{ $}}
+ ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX10-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-W64-LABEL: name: wg_rmw_add_monotonic_multi
+ ; GFX10-W64: bb.0 (%ir-block.0):
+ ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: {{ $}}
+ ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX10-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_rmw_add_monotonic_multi
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
+ ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_rmw_add_monotonic_multi
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
+ ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
+ ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
+ ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+ ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_rmw_add_monotonic_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1 (%ir-block.7):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
+ ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
+ ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2 (%ir-block.11):
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") monotonic
+ ret void
+}
+
+define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) %p, i32 %cmp, i32 %new) #1 {
+ ; GFX9-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
+ ; GFX9-NEXT: S_WAITCNT_lds_direct
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
; GFX942-NEXT: S_WAITCNT_lds_direct
; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
@@ -2152,6 +2818,168 @@ define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1)
ret void
}
+define amdgpu_kernel void @wg_cmpxchg_monotonic_monotonic_single32(ptr addrspace(1) %p, i32 %cmp, i32 %new) #0 {
+ ; GFX9-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") monotonic monotonic
+ ret void
+}
+
+define amdgpu_kernel void @wg_cmpxchg_monotonic_monotonic_multi(ptr addrspace(1) %p, i32 %cmp, i32 %new) #2 {
+ ; GFX9-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
+ ; GFX9: bb.0 (%ir-block.0):
+ ; GFX9-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: S_ENDPGM 0
+ ;
+ ; GFX942-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
+ ; GFX942: bb.0 (%ir-block.0):
+ ; GFX942-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX10-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
+ ; GFX10: bb.0 (%ir-block.0):
+ ; GFX10-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W32-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
+ ; GFX12-W32: bb.0 (%ir-block.0):
+ ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W32-NEXT: {{ $}}
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-W64-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
+ ; GFX12-W64: bb.0 (%ir-block.0):
+ ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX12-W64-NEXT: {{ $}}
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
+ ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") monotonic monotonic
+ ret void
+}
+
define amdgpu_kernel void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) %p) #0 {
; GFX9-LABEL: name: lds_wg_ld_seq_cst_single32
; GFX9: bb.0 (%ir-block.0):
>From 1a951a71f1a4a5363dd815d8beb6eaa489f63add Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Thu, 17 Sep 2026 14:26:35 -0500
Subject: [PATCH 3/3] Add `nounwind` and run tests to the end
---
...zer-single-wave-workgroup-lds-dma-async.ll | 78 +-
...legalizer-single-wave-workgroup-lds-dma.ll | 370 +-
...-legalizer-single-wave-workgroup-memops.ll | 6658 +++++++----------
3 files changed, 3138 insertions(+), 3968 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
index 13efb8daae0a3..8c521484e981f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
@@ -1,23 +1,30 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgpu12.50 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX1250
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu12.50 -O0 < %s | FileCheck %s --check-prefixes=GFX1250
declare void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) nocapture, ptr addrspace(3) nocapture, i32, i32)
define amdgpu_kernel void @lds_wg_fence_release_single32(ptr addrspace(3) %lds) #0 {
- ; GFX1250-LABEL: name: lds_wg_fence_release_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 1, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX1250-LABEL: lds_wg_fence_release_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ds_store_b32 v0, v1
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ds_load_b32 v1, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: ds_store_b32 v0, v1
+; GFX1250-NEXT: s_endpgm
store i32 1, ptr addrspace(3) %lds, align 4
fence syncscope("workgroup") release
%v = load i32, ptr addrspace(3) %lds, align 4
@@ -26,19 +33,30 @@ define amdgpu_kernel void @lds_wg_fence_release_single32(ptr addrspace(3) %lds)
}
define amdgpu_kernel void @lds_async_dma_wg_fence_release_single32(ptr addrspace(1) %g, ptr addrspace(3) %lds) #0 {
- ; GFX1250-LABEL: name: lds_async_dma_wg_fence_release_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.g.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32_SADDR killed $vgpr1, killed $sgpr0_sgpr1, killed $vgpr0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32) from %ir.g.load, align 1, addrspace 1), (store (s32) into %ir.lds.load, align 1, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX1250-LABEL: lds_async_dma_wg_fence_release_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: global_load_async_to_lds_b32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ds_load_b32 v1, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: ds_store_b32 v0, v1
+; GFX1250-NEXT: s_endpgm
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %g, ptr addrspace(3) %lds, i32 0, i32 0)
fence syncscope("workgroup") release
%v = load i32, ptr addrspace(3) %lds, align 4
@@ -46,4 +64,4 @@ define amdgpu_kernel void @lds_async_dma_wg_fence_release_single32(ptr addrspace
ret void
}
-attributes #0 = { "amdgpu-flat-work-group-size"="32,32" }
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
index 2cfe2bb0ad0bd..4d6de9716500c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
@@ -1,57 +1,63 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgpu9.00 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX9
-; RUN: llc -mtriple=amdgpu9.42 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX942
-; RUN: llc -mtriple=amdgpu10.10 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W32
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W64
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.00 -O0 < %s | FileCheck %s --check-prefixes=GFX9
+; RUN: llc -mtriple=amdgpu9.42 -O0 < %s | FileCheck %s --check-prefixes=GFX942
+; RUN: llc -mtriple=amdgpu10.10 -O0 < %s | FileCheck %s --check-prefixes=GFX10,GFX10-W32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -O0 < %s | FileCheck %s --check-prefixes=GFX10,GFX10-W64
declare void @llvm.amdgcn.raw.ptr.buffer.load.lds(ptr addrspace(8), ptr addrspace(3) nocapture, i32, i32, i32, i32, i32)
define amdgpu_kernel void @lds_wg_fence_release_single32(ptr addrspace(3) %lds) #0 {
- ; GFX9-LABEL: name: lds_wg_fence_release_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_fence_release_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_fence_release_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX10-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_fence_release_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_read_b32 v1, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_fence_release_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v1, 1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: ds_write_b32 v0, v1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: ds_read_b32 v1, v0
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_write_b32 v0, v1
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_fence_release_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: ds_write_b32 v0, v1
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: ds_read_b32 v1, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: ds_write_b32 v0, v1
+; GFX10-NEXT: s_endpgm
store i32 1, ptr addrspace(3) %lds, align 4
fence syncscope("workgroup") release
%v = load i32, ptr addrspace(3) %lds, align 4
@@ -60,51 +66,57 @@ define amdgpu_kernel void @lds_wg_fence_release_single32(ptr addrspace(3) %lds)
}
define amdgpu_kernel void @lds_wg_fence_release_single64(ptr addrspace(3) %lds) #1 {
- ; GFX9-LABEL: name: lds_wg_fence_release_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_fence_release_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_fence_release_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr1, renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: (store (s32) into %ir.lds.load, addrspace 3)
- ; GFX10-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_fence_release_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_read_b32 v1, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_fence_release_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v1, 1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: ds_write_b32 v0, v1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: ds_read_b32 v1, v0
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_write_b32 v0, v1
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_fence_release_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: ds_write_b32 v0, v1
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: ds_read_b32 v1, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: ds_write_b32 v0, v1
+; GFX10-NEXT: s_endpgm
store i32 1, ptr addrspace(3) %lds, align 4
fence syncscope("workgroup") release
%v = load i32, ptr addrspace(3) %lds, align 4
@@ -113,63 +125,110 @@ define amdgpu_kernel void @lds_wg_fence_release_single64(ptr addrspace(3) %lds)
}
define amdgpu_kernel void @lds_dma_wg_fence_release_single32(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds) #0 {
- ; GFX9-LABEL: name: lds_dma_wg_fence_release_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr6 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 52, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.rsrc.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $m0 = S_MOV_B32 killed $sgpr6
- ; GFX9-NEXT: BUFFER_LOAD_DWORD_LDS_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec, implicit $m0 :: (dereferenceable load (s32) from %ir.rsrc.load, align 1, addrspace 8), (dereferenceable store (s2048) into %ir.lds.load, align 1, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_dma_wg_fence_release_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr6 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 52, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.rsrc.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $m0 = S_MOV_B32 killed $sgpr6
- ; GFX942-NEXT: BUFFER_LOAD_DWORD_LDS_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec, implicit $m0 :: (dereferenceable load (s32) from %ir.rsrc.load, align 1, addrspace 8), (dereferenceable store (s2048) into %ir.lds.load, align 1, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: lds_dma_wg_fence_release_single32
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W32-NEXT: renamable $sgpr6 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 52, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.rsrc.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: $m0 = S_MOV_B32 killed $sgpr6
- ; GFX10-W32-NEXT: BUFFER_LOAD_DWORD_LDS_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec, implicit $m0 :: (dereferenceable load (s32) from %ir.rsrc.load, align 1, addrspace 8), (dereferenceable store (s1024) into %ir.lds.load, align 1, addrspace 3)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: lds_dma_wg_fence_release_single32
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W64-NEXT: renamable $sgpr6 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 52, 0 :: (dereferenceable invariant load (s32) from %ir.lds.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.rsrc.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: $m0 = S_MOV_B32 killed $sgpr6
- ; GFX10-W64-NEXT: BUFFER_LOAD_DWORD_LDS_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec, implicit $m0 :: (dereferenceable load (s32) from %ir.rsrc.load, align 1, addrspace 8), (dereferenceable store (s2048) into %ir.lds.load, align 1, addrspace 3)
- ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_dma_wg_fence_release_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[6:7], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[6:7], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[6:7], 0x34
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s0, 36
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[6:7], s0 offset:0x8
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: ; kill: def $sgpr2 killed $sgpr2 killed $sgpr2_sgpr3
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x24
+; GFX9-NEXT: s_load_dword s0, s[6:7], 0x34
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
+; GFX9-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_mov_b32 s6, s2
+; GFX9-NEXT: s_mov_b32 s7, s1
+; GFX9-NEXT: s_mov_b32 s1, 0
+; GFX9-NEXT: s_mov_b32 m0, s0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword off, s[4:7], s1 lds
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_read_b32 v1, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_dma_wg_fence_release_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[6:7], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[6:7], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[6:7], 0x34
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s0, 36
+; GFX942-NEXT: s_load_dwordx2 s[2:3], s[6:7], s0 offset:0x8
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s1, s3
+; GFX942-NEXT: ; kill: def $sgpr2 killed $sgpr2 killed $sgpr2_sgpr3
+; GFX942-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x24
+; GFX942-NEXT: s_load_dword s0, s[6:7], 0x34
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s3, s5
+; GFX942-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
+; GFX942-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
+; GFX942-NEXT: s_mov_b32 s5, s3
+; GFX942-NEXT: s_mov_b32 s6, s2
+; GFX942-NEXT: s_mov_b32 s7, s1
+; GFX942-NEXT: s_mov_b32 s1, 0
+; GFX942-NEXT: s_mov_b32 m0, s0
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: buffer_load_dword off, s[4:7], s1 lds
+; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: ds_read_b32 v1, v0
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_write_b32 v0, v1
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_dma_wg_fence_release_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[6:7], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[6:7], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[6:7], 0x34
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s0, 36
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[6:7], s0 offset:0x8
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: ; kill: def $sgpr2 killed $sgpr2 killed $sgpr2_sgpr3
+; GFX10-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x24
+; GFX10-NEXT: s_load_dword s0, s[6:7], 0x34
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
+; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
+; GFX10-NEXT: s_mov_b32 s5, s3
+; GFX10-NEXT: s_mov_b32 s6, s2
+; GFX10-NEXT: s_mov_b32 s7, s1
+; GFX10-NEXT: s_mov_b32 s1, 0
+; GFX10-NEXT: s_mov_b32 m0, s0
+; GFX10-NEXT: buffer_load_dword off, s[4:7], s1 lds
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: ds_read_b32 v1, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: ds_write_b32 v0, v1
+; GFX10-NEXT: s_endpgm
call void @llvm.amdgcn.raw.ptr.buffer.load.lds(ptr addrspace(8) %rsrc, ptr addrspace(3) %lds, i32 4, i32 0, i32 0, i32 0, i32 0)
fence syncscope("workgroup") release
%v = load i32, ptr addrspace(3) %lds, align 4
@@ -177,5 +236,8 @@ define amdgpu_kernel void @lds_dma_wg_fence_release_single32(ptr addrspace(8) %r
ret void
}
-attributes #0 = { "amdgpu-flat-work-group-size"="32,32" }
-attributes #1 = { "amdgpu-flat-work-group-size"="64,64" }
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" }
+attributes #1 = { nounwind "amdgpu-flat-work-group-size"="64,64" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10-W32: {{.*}}
+; GFX10-W64: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
index 8d465f045a993..1abe56fba5c69 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
@@ -1,3986 +1,3076 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgpu9.00 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX9
-; RUN: llc -mtriple=amdgpu9.42 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX942
-; RUN: llc -mtriple=amdgpu10.10 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W32
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W64
-; RUN: llc -mtriple=amdgpu12.00 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX12,GFX12-W32
-; RUN: llc -mtriple=amdgpu12.00 -mattr=+wavefrontsize64 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX12,GFX12-W64
-; RUN: llc -mtriple=amdgpu12.50 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX1250
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.00 -O0 < %s | FileCheck %s --check-prefixes=GFX9
+; RUN: llc -mtriple=amdgpu9.42 -O0 < %s | FileCheck %s --check-prefixes=GFX942
+; RUN: llc -mtriple=amdgpu10.10 -O0 < %s | FileCheck %s --check-prefixes=GFX10,GFX10-W32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -O0 < %s | FileCheck %s --check-prefixes=GFX10,GFX10-W64
+; RUN: llc -mtriple=amdgpu12.00 -O0 < %s | FileCheck %s --check-prefixes=GFX12,GFX12-W32
+; RUN: llc -mtriple=amdgpu12.00 -mattr=+wavefrontsize64 -O0 < %s | FileCheck %s --check-prefixes=GFX12,GFX12-W64
+; RUN: llc -mtriple=amdgpu12.50 -O0 < %s | FileCheck %s --check-prefixes=GFX1250
define amdgpu_kernel void @wg_fence_acq_rel_single32() #0 {
- ; GFX9-LABEL: name: wg_fence_acq_rel_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_acq_rel_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_fence_acq_rel_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_fence_acq_rel_single32
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_acq_rel_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_fence_acq_rel_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_fence_acq_rel_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_fence_acq_rel_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_fence_acq_rel_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_fence_acq_rel_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
fence syncscope("workgroup") acq_rel
ret void
}
define amdgpu_kernel void @wg_fence_acq_rel_single64() #1 {
- ; GFX9-LABEL: name: wg_fence_acq_rel_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_acq_rel_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_fence_acq_rel_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_fence_acq_rel_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_acq_rel_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_fence_acq_rel_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_fence_acq_rel_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_fence_acq_rel_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_fence_acq_rel_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_fence_acq_rel_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
fence syncscope("workgroup") acq_rel
ret void
}
define amdgpu_kernel void @wg_fence_acq_rel_multi() #2 {
- ; GFX9-LABEL: name: wg_fence_acq_rel_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_acq_rel_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_fence_acq_rel_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_fence_acq_rel_multi
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_acq_rel_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_fence_acq_rel_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_fence_acq_rel_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_fence_acq_rel_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_fence_acq_rel_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_fence_acq_rel_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
fence syncscope("workgroup") acq_rel
ret void
}
define amdgpu_kernel void @wg_fence_acquire_single64() #1 {
- ; GFX9-LABEL: name: wg_fence_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_fence_acquire_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_fence_acquire_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_fence_acquire_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_fence_acquire_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_fence_acquire_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_fence_acquire_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_fence_acquire_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
fence syncscope("workgroup") acquire
ret void
}
define amdgpu_kernel void @wg_fence_release_single64() #1 {
- ; GFX9-LABEL: name: wg_fence_release_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_release_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_fence_release_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_fence_release_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_release_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_fence_release_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_fence_release_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_fence_release_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_fence_release_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_fence_release_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
fence syncscope("workgroup") release
ret void
}
define amdgpu_kernel void @wg_fence_seq_cst_single64() #1 {
- ; GFX9-LABEL: name: wg_fence_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_fence_seq_cst_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_fence_seq_cst_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_fence_seq_cst_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_fence_seq_cst_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_fence_seq_cst_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_fence_seq_cst_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_fence_seq_cst_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
fence syncscope("workgroup") seq_cst
ret void
}
define amdgpu_kernel void @wg_ld_seq_cst_single32(ptr addrspace(1) %p) #0 {
- ; GFX9-LABEL: name: wg_ld_seq_cst_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_seq_cst_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_seq_cst_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_seq_cst_single32
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_seq_cst_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_ld_seq_cst_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_ld_seq_cst_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_load_dword v0, v0, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_ld_seq_cst_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_load_dword v0, v0, s[0:1] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_ld_seq_cst_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_ld_seq_cst_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @wg_ld_seq_cst_single64(ptr addrspace(1) %p) #1 {
- ; GFX9-LABEL: name: wg_ld_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_seq_cst_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_seq_cst_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_ld_seq_cst_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_ld_seq_cst_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_load_dword v0, v0, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_ld_seq_cst_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_load_dword v0, v0, s[0:1] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_ld_seq_cst_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_ld_seq_cst_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @wg_ld_seq_cst_multi(ptr addrspace(1) %p) #2 {
- ; GFX9-LABEL: name: wg_ld_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_seq_cst_multi
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_ld_seq_cst_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_ld_seq_cst_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_load_dword v0, v0, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_ld_seq_cst_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_load_dword v0, v0, s[0:1] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_ld_seq_cst_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_ld_seq_cst_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @wg_ld_acquire_single64(ptr addrspace(1) %p) #1 {
- ; GFX9-LABEL: name: wg_ld_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_acquire_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_acquire_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_ld_acquire_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_ld_acquire_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_load_dword v0, v0, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_ld_acquire_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v0, v0, s[0:1] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_ld_acquire_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_ld_acquire_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") acquire, align 4
ret void
}
define amdgpu_kernel void @wg_ld_monotonic_single32(ptr addrspace(1) %p) #0 {
- ; GFX9-LABEL: name: wg_ld_monotonic_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_monotonic_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_monotonic_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_monotonic_single32
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_monotonic_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_ld_monotonic_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_ld_monotonic_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_load_dword v0, v0, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_ld_monotonic_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v0, v0, s[0:1] glc
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_ld_monotonic_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_ld_monotonic_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
ret void
}
define amdgpu_kernel void @wg_ld_monotonic_single64(ptr addrspace(1) %p) #1 {
- ; GFX9-LABEL: name: wg_ld_monotonic_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_monotonic_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_monotonic_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_monotonic_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_monotonic_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_ld_monotonic_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_ld_monotonic_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_load_dword v0, v0, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_ld_monotonic_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v0, v0, s[0:1] glc
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_ld_monotonic_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_ld_monotonic_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
ret void
}
define amdgpu_kernel void @wg_ld_monotonic_multi(ptr addrspace(1) %p) #2 {
- ; GFX9-LABEL: name: wg_ld_monotonic_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_monotonic_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_monotonic_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_monotonic_multi
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_monotonic_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_ld_monotonic_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_ld_monotonic_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_load_dword v0, v0, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_ld_monotonic_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v0, v0, s[0:1] glc
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_ld_monotonic_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_ld_monotonic_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
ret void
}
define amdgpu_kernel void @wg_st_seq_cst_single32(ptr addrspace(1) %p, i32 %x) #0 {
- ; GFX9-LABEL: name: wg_st_seq_cst_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_seq_cst_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_st_seq_cst_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_seq_cst_single32
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_seq_cst_single32
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_seq_cst_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_st_seq_cst_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_st_seq_cst_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_st_seq_cst_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_st_seq_cst_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_st_seq_cst_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @wg_st_seq_cst_single64(ptr addrspace(1) %p, i32 %x) #1 {
- ; GFX9-LABEL: name: wg_st_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_st_seq_cst_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_seq_cst_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_seq_cst_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_st_seq_cst_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_st_seq_cst_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_st_seq_cst_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_st_seq_cst_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_st_seq_cst_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @wg_st_seq_cst_multi(ptr addrspace(1) %p, i32 %x) #2 {
- ; GFX9-LABEL: name: wg_st_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_st_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_seq_cst_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_seq_cst_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_st_seq_cst_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_st_seq_cst_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_st_seq_cst_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_st_seq_cst_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_st_seq_cst_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @wg_st_release_single64(ptr addrspace(1) %p, i32 %x) #1 {
- ; GFX9-LABEL: name: wg_st_release_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_release_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_st_release_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_release_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_release_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_release_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_st_release_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_st_release_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_st_release_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_st_release_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_st_release_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") release, align 4
ret void
}
define amdgpu_kernel void @wg_st_monotonic_single32(ptr addrspace(1) %p, i32 %x) #0 {
- ; GFX9-LABEL: name: wg_st_monotonic_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_monotonic_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_st_monotonic_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_monotonic_single32
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_monotonic_single32
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_monotonic_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_st_monotonic_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_st_monotonic_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_st_monotonic_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_st_monotonic_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_st_monotonic_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
ret void
}
define amdgpu_kernel void @wg_st_monotonic_multi(ptr addrspace(1) %p, i32 %x) #2 {
- ; GFX9-LABEL: name: wg_st_monotonic_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_monotonic_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_st_monotonic_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_monotonic_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_monotonic_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_monotonic_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") monotonic (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_st_monotonic_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_st_monotonic_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_st_monotonic_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_st_monotonic_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_st_monotonic_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
ret void
}
define amdgpu_kernel void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) %p) #0 {
- ; GFX9-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_rmw_add_seq_cst_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 7
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_rmw_add_seq_cst_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v1, 7
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_rmw_add_seq_cst_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_rmw_add_seq_cst_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_rmw_add_seq_cst_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 7
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
ret void
}
define amdgpu_kernel void @wg_rmw_add_seq_cst_single64(ptr addrspace(1) %p) #1 {
- ; GFX9-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 7
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v1, 7
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 7
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
ret void
}
define amdgpu_kernel void @wg_rmw_add_seq_cst_multi(ptr addrspace(1) %p) #2 {
- ; GFX9-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_rmw_add_seq_cst_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 7
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_rmw_add_seq_cst_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v1, 7
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_rmw_add_seq_cst_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_rmw_add_seq_cst_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_rmw_add_seq_cst_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 7
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
ret void
}
define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32 %x) #1 {
- ; GFX9-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_atomic_swap v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_atomic_swap v0, v1, s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_swap v0, v1, s[0:1]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = atomicrmw xchg ptr addrspace(1) %p, i32 %x syncscope("workgroup") acq_rel
ret void
}
define amdgpu_kernel void @wg_rmw_add_monotonic_single32(ptr addrspace(1) %p) #0 {
- ; GFX9-LABEL: name: wg_rmw_add_monotonic_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_add_monotonic_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_add_monotonic_single32
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_add_monotonic_single32
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_add_monotonic_single32
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_add_monotonic_single32
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_add_monotonic_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_rmw_add_monotonic_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 7
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_rmw_add_monotonic_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v1, 7
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_rmw_add_monotonic_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_rmw_add_monotonic_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_rmw_add_monotonic_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 7
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") monotonic
ret void
}
define amdgpu_kernel void @wg_rmw_add_monotonic_multi(ptr addrspace(1) %p) #2 {
- ; GFX9-LABEL: name: wg_rmw_add_monotonic_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_add_monotonic_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_add_monotonic_multi
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_add_monotonic_multi
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_add_monotonic_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_add_monotonic_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_add_monotonic_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_rmw_add_monotonic_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 7
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_rmw_add_monotonic_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v1, 7
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_rmw_add_monotonic_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_atomic_add v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_rmw_add_monotonic_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_rmw_add_monotonic_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 7
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") monotonic
ret void
}
define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX9-NEXT: v_mov_b32_e32 v2, v3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x30
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b32 s3, s[4:5], 0x30
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v3, s2
+; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x30 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], s[0:1]
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") acq_rel monotonic
ret void
}
define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p, i32 %cmp, i32 %new) #2 {
- ; GFX9-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_cmpxchg_seq_cst_seq_cst_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX9-NEXT: v_mov_b32_e32 v2, v3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_cmpxchg_seq_cst_seq_cst_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_cmpxchg_seq_cst_seq_cst_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_cmpxchg_seq_cst_seq_cst_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x30
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b32 s3, s[4:5], 0x30
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v3, s2
+; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_cmpxchg_seq_cst_seq_cst_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x30 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], s[0:1]
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") seq_cst seq_cst
ret void
}
define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX9-NEXT: v_mov_b32_e32 v2, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x30
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b32 s3, s[4:5], 0x30
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v3, s2
+; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x30 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], s[0:1]
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") acquire acquire
ret void
}
define amdgpu_kernel void @wg_cmpxchg_monotonic_monotonic_single32(ptr addrspace(1) %p, i32 %cmp, i32 %new) #0 {
- ; GFX9-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_cmpxchg_monotonic_monotonic_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_cmpxchg_monotonic_monotonic_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX9-NEXT: v_mov_b32_e32 v2, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_cmpxchg_monotonic_monotonic_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_cmpxchg_monotonic_monotonic_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_cmpxchg_monotonic_monotonic_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x30
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b32 s3, s[4:5], 0x30
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v3, s2
+; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_cmpxchg_monotonic_monotonic_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x30 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], s[0:1]
+; GFX1250-NEXT: s_endpgm
%r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") monotonic monotonic
ret void
}
define amdgpu_kernel void @wg_cmpxchg_monotonic_monotonic_multi(ptr addrspace(1) %p, i32 %cmp, i32 %new) #2 {
- ; GFX9-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_cmpxchg_monotonic_monotonic_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: wg_cmpxchg_monotonic_monotonic_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX9-NEXT: v_mov_b32_e32 v2, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: wg_cmpxchg_monotonic_monotonic_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: wg_cmpxchg_monotonic_monotonic_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x30
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dword s3, s[4:5], 0x30
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: wg_cmpxchg_monotonic_monotonic_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x30
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b32 s3, s[4:5], 0x30
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v3, s2
+; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: wg_cmpxchg_monotonic_monotonic_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x30 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], s[0:1]
+; GFX1250-NEXT: s_endpgm
%r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") monotonic monotonic
ret void
}
define amdgpu_kernel void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) %p) #0 {
- ; GFX9-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_ld_seq_cst_single32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_ld_seq_cst_single32:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_read_b32 v0, v0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_ld_seq_cst_single32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: lds_wg_ld_seq_cst_single32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: ds_load_b32 v0, v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: lds_wg_ld_seq_cst_single32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: ds_load_b32 v0, v0
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @lds_wg_ld_seq_cst_single64(ptr addrspace(3) %p) #1 {
- ; GFX9-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_read_b32 v0, v0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: ds_load_b32 v0, v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: ds_load_b32 v0, v0
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @lds_wg_ld_seq_cst_multi(ptr addrspace(3) %p) #2 {
- ; GFX9-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_ld_seq_cst_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_ld_seq_cst_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_read_b32 v0, v0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_ld_seq_cst_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: lds_wg_ld_seq_cst_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: ds_load_b32 v0, v0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: lds_wg_ld_seq_cst_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: ds_load_b32 v0, v0
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
ret void
}
define amdgpu_kernel void @lds_wg_st_release_single64(ptr addrspace(3) %p, i32 %x) #1 {
- ; GFX9-LABEL: name: lds_wg_st_release_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_st_release_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_st_release_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_st_release_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_st_release_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_st_release_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_st_release_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX9-NEXT: s_load_dword s1, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_st_release_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX942-NEXT: s_load_dword s1, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX942-NEXT: v_mov_b32_e32 v0, s1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_write_b32 v0, v1
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_st_release_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX10-NEXT: s_load_dword s1, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX10-NEXT: v_mov_b32_e32 v0, s1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b32 v0, v1
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: lds_wg_st_release_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x28
+; GFX12-NEXT: s_load_b32 s1, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x28
+; GFX12-NEXT: v_mov_b32_e32 v0, s1
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: ds_store_b32 v0, v1
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: lds_wg_st_release_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x28 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, s1
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: ds_store_b32 v0, v1
+; GFX1250-NEXT: s_endpgm
store atomic i32 %x, ptr addrspace(3) %p syncscope("workgroup") release, align 4
ret void
}
define amdgpu_kernel void @lds_wg_rmw_add_acq_rel_single64(ptr addrspace(3) %p) #1 {
- ; GFX9-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX10-W64-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX12-W32-NEXT: $vgpr1, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX1250-NEXT: $vgpr1, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_add_u32 v0, v1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v1, 3
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_add_u32 v0, v1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 3
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_add_u32 v0, v1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v1, 3
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: ds_add_u32 v0, v1
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: ds_add_u32 v0, v1
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = atomicrmw add ptr addrspace(3) %p, i32 3 syncscope("workgroup") acq_rel
ret void
}
define amdgpu_kernel void @lds_wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(3) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s1, s[4:5], 0x28
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s1, s[4:5], 0x28
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, s2
+; GFX942-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s1, s[4:5], 0x28
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x28
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x28
+; GFX12-NEXT: s_load_b32 s1, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s1
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = cmpxchg ptr addrspace(3) %p, i32 %cmp, i32 %new syncscope("workgroup") acq_rel monotonic
ret void
}
define amdgpu_kernel void @lds_wg_ld_unordered_single64(ptr addrspace(3) %p) #1 {
- ; GFX9-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_ld_unordered_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_read_b32 v0, v0
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_ld_unordered_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: ds_read_b32 v0, v0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_ld_unordered_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: ds_read_b32 v0, v0
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: lds_wg_ld_unordered_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ds_load_b32 v0, v0
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: lds_wg_ld_unordered_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ds_load_b32 v0, v0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") unordered, align 4
ret void
}
define amdgpu_kernel void @lds_wg_cmpxchg_monotonic_acquire_single64(ptr addrspace(3) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX1250-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s1, s[4:5], 0x28
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX942-NEXT: s_load_dword s1, s[4:5], 0x28
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b32_e32 v0, s2
+; GFX942-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s0
+; GFX942-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x28
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s1, s[4:5], 0x28
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x28
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x28
+; GFX12-NEXT: s_load_b32 s1, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s1
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
%r = cmpxchg ptr addrspace(3) %p, i32 %cmp, i32 %new syncscope("workgroup") monotonic acquire
ret void
}
define amdgpu_kernel void @flat_wg_ld_acquire_single64(ptr addrspace(0) %p) #1 {
- ; GFX9-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 killed $sgpr0_sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 1, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX10-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 1, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX12-W32-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 8, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX12-W64-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 8, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: flat_wg_ld_acquire_single64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: flat_load_dword v0, v[0:1]
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: flat_wg_ld_acquire_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-NEXT: flat_load_dword v0, v[0:1] sc0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: flat_wg_ld_acquire_single64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: flat_wg_ld_acquire_single64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: flat_wg_ld_acquire_single64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: flat_load_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_endpgm
%v = load atomic i32, ptr addrspace(0) %p syncscope("workgroup") acquire, align 4
ret void
}
define amdgpu_kernel void @flat_wg_st_seq_cst_multi(ptr addrspace(0) %p, i32 %x) #2 {
- ; GFX9-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX9-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX942-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 killed $sgpr0_sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 1, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX10-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX10-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W32-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX12-W64-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x04, 0x30, 0x36, 0xe9, 0x02
- ; GFX1250-NEXT: frame-setup CFI_INSTRUCTION undefined $pc_reg
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: FLAT_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX1250-NEXT: S_ENDPGM 0
+; GFX9-LABEL: flat_wg_st_seq_cst_multi:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: flat_store_dword v[0:1], v2
+; GFX9-NEXT: s_endpgm
+;
+; GFX942-LABEL: flat_wg_st_seq_cst_multi:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: flat_store_dword v[0:1], v2 sc0
+; GFX942-NEXT: s_endpgm
+;
+; GFX10-LABEL: flat_wg_st_seq_cst_multi:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: flat_store_dword v[0:1], v2
+; GFX10-NEXT: s_endpgm
+;
+; GFX12-LABEL: flat_wg_st_seq_cst_multi:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE
+; GFX12-NEXT: s_endpgm
+;
+; GFX1250-LABEL: flat_wg_st_seq_cst_multi:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: flat_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
store atomic i32 %x, ptr addrspace(0) %p syncscope("workgroup") seq_cst, align 4
ret void
}
-attributes #0 = { "amdgpu-flat-work-group-size"="32,32" }
-attributes #1 = { "amdgpu-flat-work-group-size"="64,64" }
-attributes #2 = { "amdgpu-flat-work-group-size"="64,256" }
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" }
+attributes #1 = { nounwind "amdgpu-flat-work-group-size"="64,64" }
+attributes #2 = { nounwind "amdgpu-flat-work-group-size"="64,256" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10-W32: {{.*}}
+; GFX10-W64: {{.*}}
+; GFX12-W32: {{.*}}
+; GFX12-W64: {{.*}}
More information about the llvm-commits
mailing list