[llvm] [AMDGPU] Precommit lit test for optimized xcnt insertion. (PR #177620)
Christudasan Devadasan via llvm-commits
llvm-commits at lists.llvm.org
Fri Jan 23 09:18:48 PST 2026
https://github.com/cdevadas updated https://github.com/llvm/llvm-project/pull/177620
>From 82164b6521da60fdd7032ceff4f296b4ea3fd97b Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <Christudasan.Devadasan at amd.com>
Date: Fri, 23 Jan 2026 14:38:43 +0000
Subject: [PATCH 1/2] [AMDGPU] Precommit lit test for optimized xcnt insertion.
---
.../wait-xcnt-atomic-rmw-optimization.ll | 1302 +++++++++++++++++
1 file changed, 1302 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
new file mode 100644
index 0000000000000..f699a6b73798f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
@@ -0,0 +1,1302 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
+
+; Test single atomic RMW - s_wait_xcnt should be kept.
+define amdgpu_kernel void @single_atomic_rmw(ptr addrspace(1) %ptr) {
+; GFX1250-LABEL: single_atomic_rmw:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ ret void
+}
+
+; Test back-to-back atomic RMW operations - only first s_wait_xcnt should remain.
+define amdgpu_kernel void @atomic_rmw_back_to_back(ptr addrspace(1) %ptr) {
+; GFX1250-LABEL: atomic_rmw_back_to_back:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ ret void
+}
+
+; Test atomic RMW block with interleaved ALU ops - should not break the block.
+define amdgpu_kernel void @atomic_rmw_with_alu(ptr addrspace(1) %ptr, i32 %a, i32 %b) {
+; GFX1250-LABEL: atomic_rmw_with_alu:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0xc
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_add_co_i32 s4, s2, s3
+; GFX1250-NEXT: v_mov_b32_e32 v1, s4
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mul_i32 s2, s2, s3
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %sum = add i32 %a, %b
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 %sum seq_cst
+ %prod = mul i32 %a, %b
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 %prod seq_cst
+ ret void
+}
+
+; Test atomic RMW block broken by global load (VMEM).
+define amdgpu_kernel void @atomic_rmw_broken_by_global_load(ptr addrspace(1) %ptr, ptr addrspace(1) %load_ptr) {
+; GFX1250-LABEL: atomic_rmw_broken_by_global_load:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %load = load i32, ptr addrspace(1) %load_ptr
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 %load seq_cst
+ ret void
+}
+
+; Test atomic RMW block broken by global store (VMEM).
+define amdgpu_kernel void @atomic_rmw_broken_by_global_store(ptr addrspace(1) %ptr, ptr addrspace(1) %store_ptr) {
+; GFX1250-LABEL: atomic_rmw_broken_by_global_store:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 42
+; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ store i32 42, ptr addrspace(1) %store_ptr
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ ret void
+}
+
+; Test atomic RMW block broken by FLAT load (VMEM).
+define amdgpu_kernel void @atomic_rmw_broken_by_flat_load(ptr addrspace(1) %ptr, ptr %flat_ptr) {
+; GFX1250-LABEL: atomic_rmw_broken_by_flat_load:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %load = load i32, ptr %flat_ptr, align 4
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 %load seq_cst
+ ret void
+}
+
+; Test atomic RMW block broken by FLAT store (VMEM).
+define amdgpu_kernel void @atomic_rmw_broken_by_flat_store(ptr addrspace(1) %ptr, ptr %flat_ptr) {
+; GFX1250-LABEL: atomic_rmw_broken_by_flat_store:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 42
+; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ store i32 42, ptr %flat_ptr, align 4
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ ret void
+}
+
+; Test atomic RMW block broken by SMEM load.
+define amdgpu_kernel void @atomic_rmw_broken_by_smem_load(ptr addrspace(1) %ptr, ptr addrspace(4) %const_ptr) {
+; GFX1250-LABEL: atomic_rmw_broken_by_smem_load:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %load = load i32, ptr addrspace(4) %const_ptr
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 %load seq_cst
+ ret void
+}
+
+; Test atomic RMW block broken by atomic store.
+define amdgpu_kernel void @atomic_rmw_broken_by_atomic_store(ptr addrspace(1) %ptr, ptr addrspace(1) %store_ptr) {
+; GFX1250-LABEL: atomic_rmw_broken_by_atomic_store:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 42
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ store atomic i32 42, ptr addrspace(1) %store_ptr seq_cst, align 4
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ ret void
+}
+
+; Test LDS load should not break atomic RMW block.
+define amdgpu_kernel void @atomic_rmw_with_lds_load(ptr addrspace(1) %ptr, ptr addrspace(3) %lds_ptr) {
+; GFX1250-LABEL: atomic_rmw_with_lds_load:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: ds_load_b32 v1, v1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %load = load i32, ptr addrspace(3) %lds_ptr, align 4
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 %load seq_cst
+ ret void
+}
+
+; Test LDS store should not break atomic RMW block.
+define amdgpu_kernel void @atomic_rmw_with_lds_store(ptr addrspace(1) %ptr, ptr addrspace(3) %lds_ptr) {
+; GFX1250-LABEL: atomic_rmw_with_lds_store:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, 42
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: ds_store_b32 v1, v2
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ store i32 42, ptr addrspace(3) %lds_ptr, align 4
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ ret void
+}
+
+; Test FLAT load from LDS should not break atomic RMW block.
+define amdgpu_kernel void @atomic_rmw_with_flat_lds_load(ptr addrspace(1) %ptr, ptr addrspace(3) %lds_ptr) {
+; GFX1250-LABEL: atomic_rmw_with_flat_lds_load:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[6:7], 0
+; GFX1250-NEXT: s_mov_b32 s4, s7
+; GFX1250-NEXT: s_mov_b64 s[8:9], src_shared_base
+; GFX1250-NEXT: s_mov_b32 s3, s9
+; GFX1250-NEXT: s_mov_b32 s5, -1
+; GFX1250-NEXT: s_cmp_lg_u32 s2, s5
+; GFX1250-NEXT: s_cselect_b32 s4, s3, s4
+; GFX1250-NEXT: s_mov_b32 s3, s6
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s3
+; GFX1250-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s3, s4
+; GFX1250-NEXT: flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %flat_lds = addrspacecast ptr addrspace(3) %lds_ptr to ptr
+ %load = load i32, ptr %flat_lds, align 4
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 %load seq_cst
+ ret void
+}
+
+; Test FLAT store to LDS should not break atomic RMW block.
+define amdgpu_kernel void @atomic_rmw_with_flat_lds_store(ptr addrspace(1) %ptr, ptr addrspace(3) %lds_ptr) {
+; GFX1250-LABEL: atomic_rmw_with_flat_lds_store:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[6:7], 0
+; GFX1250-NEXT: s_mov_b32 s4, s7
+; GFX1250-NEXT: s_mov_b64 s[8:9], src_shared_base
+; GFX1250-NEXT: s_mov_b32 s3, s9
+; GFX1250-NEXT: s_mov_b32 s5, -1
+; GFX1250-NEXT: s_cmp_lg_u32 s2, s5
+; GFX1250-NEXT: s_cselect_b32 s4, s3, s4
+; GFX1250-NEXT: s_mov_b32 s3, s6
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s3
+; GFX1250-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s3, s4
+; GFX1250-NEXT: v_mov_b32_e32 v1, 42
+; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %flat_lds = addrspacecast ptr addrspace(3) %lds_ptr to ptr
+ store i32 42, ptr %flat_lds, align 4
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ ret void
+}
+
+; Test atomic RMW block broken by async copy from global to LDS.
+define amdgpu_kernel void @atomic_rmw_borken_by_async_lds_copy(ptr addrspace(1) %ptr, ptr addrspace(1) %src, ptr addrspace(3) %dst) {
+; GFX1250-LABEL: atomic_rmw_borken_by_async_lds_copy:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_mov_b64 s[2:3], s[4:5]
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
+; GFX1250-NEXT: s_load_b64 s[4:5], s[2:3], 0x8
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[2:3], 0x10
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_load_b32 v2, v0, s[4:5]
+; GFX1250-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ds_store_b32 v1, v2
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %load = load i32, ptr addrspace(1) %src, align 4
+ store i32 %load, ptr addrspace(3) %dst, align 4
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ ret void
+}
+
+; Test multiple separate atomic RMW blocks.
+define amdgpu_kernel void @multiple_atomic_rmw_blocks(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2) {
+ ; First block
+; GFX1250-LABEL: multiple_atomic_rmw_blocks:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 4
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr1, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr1, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr1, i64 2
+ %gep4 = getelementptr i32, ptr addrspace(1) %ptr1, i64 3
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %load = load i32, ptr addrspace(1) %ptr2
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 %load seq_cst
+ %val4 = atomicrmw add ptr addrspace(1) %gep4, i32 4 seq_cst
+ ret void
+}
+
+; Test different atomic RMW operations in a block.
+define amdgpu_kernel void @different_atomic_ops(ptr addrspace(1) %ptr) {
+; GFX1250-LABEL: different_atomic_ops:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_sub_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_and_b32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 4
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_or_b32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %gep4 = getelementptr i32, ptr addrspace(1) %ptr, i64 3
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw sub ptr addrspace(1) %gep2, i32 2 seq_cst
+ %val3 = atomicrmw and ptr addrspace(1) %gep3, i32 3 seq_cst
+ %val4 = atomicrmw or ptr addrspace(1) %gep4, i32 4 seq_cst
+ ret void
+}
+
+; Test atomic RMW block reset at basic block boundary.
+define amdgpu_kernel void @atomic_rmw_across_basic_blocks(ptr addrspace(1) %ptr, i32 %cond) {
+; GFX1250-LABEL: atomic_rmw_across_basic_blocks:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
+; GFX1250-NEXT: v_writelane_b32 v2, s4, 0
+; GFX1250-NEXT: v_writelane_b32 v2, s5, 1
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_store_b32 off, v2, off ; 4-byte Folded Spill
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s0, s1
+; GFX1250-NEXT: s_cbranch_scc1 .LBB16_2
+; GFX1250-NEXT: ; %bb.1: ; %then
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_load_b32 v2, off, off ; 4-byte Folded Reload
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: .LBB16_2: ; %exit
+; GFX1250-NEXT: s_endpgm
+entry:
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %cmp = icmp eq i32 %cond, 0
+ br i1 %cmp, label %then, label %exit
+
+then:
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ br label %exit
+
+exit:
+ ret void
+}
+
+; Test atomic RMW block in loop.
+define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
+; GFX1250-LABEL: atomic_rmw_in_loop:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_writelane_b32 v2, s2, 0
+; GFX1250-NEXT: v_writelane_b32 v2, s3, 1
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: v_writelane_b32 v2, s1, 2
+; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_store_b32 off, v2, off ; 4-byte Folded Spill
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: .LBB17_1: ; %loop
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_load_b32 v2, off, off ; 4-byte Folded Reload
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 3
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 2
+; GFX1250-NEXT: v_readlane_b32 s2, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s3, v2, 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s2, 1
+; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
+; GFX1250-NEXT: s_cmp_lt_u32 s0, s1
+; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
+; GFX1250-NEXT: s_mov_b32 s6, exec_lo
+; GFX1250-NEXT: s_mov_b32 exec_lo, -1
+; GFX1250-NEXT: scratch_store_b32 off, v2, off ; 4-byte Folded Spill
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_cbranch_scc1 .LBB17_1
+; GFX1250-NEXT: ; %bb.2: ; %exit
+; GFX1250-NEXT: s_endpgm
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 %i seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 %i seq_cst
+ %i.next = add i32 %i, 1
+ %cmp = icmp ult i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test atomic RMW block with branch in between - state reset at new block.
+define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %cond) {
+; GFX1250-LABEL: atomic_rmw_with_branch:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
+; GFX1250-NEXT: v_writelane_b32 v2, s4, 0
+; GFX1250-NEXT: v_writelane_b32 v2, s5, 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s0, -1
+; GFX1250-NEXT: s_mov_b32 s2, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s1, s2
+; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
+; GFX1250-NEXT: s_mov_b32 s6, exec_lo
+; GFX1250-NEXT: s_mov_b32 exec_lo, -1
+; GFX1250-NEXT: scratch_store_b32 off, v2, off ; 4-byte Folded Spill
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_cbranch_scc1 .LBB18_3
+; GFX1250-NEXT: .LBB18_1: ; %Flow
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_load_b32 v2, off, off ; 4-byte Folded Reload
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 2
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX1250-NEXT: s_mov_b32 s0, 1
+; GFX1250-NEXT: v_cmp_ne_u32_e64 s0, v0, s0
+; GFX1250-NEXT: s_and_b32 vcc_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_vccnz .LBB18_4
+; GFX1250-NEXT: ; %bb.2: ; %bb1
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_load_b32 v2, off, off ; 4-byte Folded Reload
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_branch .LBB18_4
+; GFX1250-NEXT: .LBB18_3: ; %bb2
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_load_b32 v2, off, off ; 4-byte Folded Reload
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 4
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_store_b32 off, v2, off ; 4-byte Folded Spill
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_branch .LBB18_1
+; GFX1250-NEXT: .LBB18_4: ; %merge
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_load_b32 v2, off, off ; 4-byte Folded Reload
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 5
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+entry:
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ %cmp = icmp eq i32 %cond, 0
+ br i1 %cmp, label %bb1, label %bb2
+
+bb1:
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ br label %merge
+
+bb2:
+ %gep4 = getelementptr i32, ptr addrspace(1) %ptr, i64 3
+ %val4 = atomicrmw add ptr addrspace(1) %gep4, i32 4 seq_cst
+ br label %merge
+
+merge:
+ %gep5 = getelementptr i32, ptr addrspace(1) %ptr, i64 4
+ %val5 = atomicrmw add ptr addrspace(1) %gep5, i32 5 seq_cst
+ ret void
+}
+
+; Test fall-through block.
+define amdgpu_kernel void @atomic_rmw_fallthrough(ptr addrspace(1) %ptr) {
+; GFX1250-LABEL: atomic_rmw_fallthrough:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
+; GFX1250-NEXT: v_writelane_b32 v2, s2, 0
+; GFX1250-NEXT: v_writelane_b32 v2, s3, 1
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_store_b32 off, v2, off ; 4-byte Folded Spill
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; %bb.1: ; %next
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_load_b32 v2, off, off ; 4-byte Folded Reload
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 exec_lo, s6
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 3
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 4
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+entry:
+ %gep1 = getelementptr i32, ptr addrspace(1) %ptr, i64 0
+ %gep2 = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+ %val1 = atomicrmw add ptr addrspace(1) %gep1, i32 1 seq_cst
+ %val2 = atomicrmw add ptr addrspace(1) %gep2, i32 2 seq_cst
+ br label %next
+
+next:
+ %gep3 = getelementptr i32, ptr addrspace(1) %ptr, i64 2
+ %gep4 = getelementptr i32, ptr addrspace(1) %ptr, i64 3
+ %val3 = atomicrmw add ptr addrspace(1) %gep3, i32 3 seq_cst
+ %val4 = atomicrmw add ptr addrspace(1) %gep4, i32 4 seq_cst
+ ret void
+}
>From cc3095ad30ddeacecdbd1f891a8a698d090370ff Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <Christudasan.Devadasan at amd.com>
Date: Fri, 23 Jan 2026 14:43:12 +0000
Subject: [PATCH 2/2] [AMDGPU][GFX1250] Optimize s_wait_xcnt for back-to-back
atomic RMWs
This patch optimizes the insertion of s_wait_xcnt instruction for
sequences of atomic read-modify-write (RMW) operations in the
SIInsertWaitcnts pass. The Memory Legalizer conservatively inserts a
soft xcnt instruction before each atomic RMW operation as part of PR
However, for back-to-back atomic RMWs, only the first s_wait_xcnt is
necessary for better runtime performance. This patch tracks atomic
RMW blocks within each basic block and removes redundant soft xcnt
instructions, keeping only the first wait in each sequence. An atomic
RMW block continues through subsequent atomic RMWs and non-memory
instructions (e.g., ALU operations) but is broken by CU-scoped memory
operations, atomic stores, or basic block boundaries.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 73 ++++++++++++++++++-
.../wait-xcnt-atomic-rmw-optimization.ll | 31 --------
2 files changed, 71 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 349c158c8838d..11e7808528fc2 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -134,6 +134,21 @@ static unsigned getWaitCountMax(const AMDGPU::HardwareLimits &Limits,
}
}
+static bool isSoftXcnt(MachineInstr &MI) {
+ return MI.getOpcode() == AMDGPU::S_WAIT_XCNT_soft;
+}
+
+static bool isAtomicRMW(MachineInstr &MI) {
+ return (MI.getDesc().TSFlags & SIInstrFlags::maybeAtomic) && MI.mayLoad() &&
+ MI.mayStore();
+}
+
+enum class AtomicRMWState {
+ NewBlock, // Start of a new atomic RMW block
+ InsideBlock, // Middle of an existing block
+ NotInBlock // Not in an atomic RMW block
+};
+
/// Integer IDs used to track vector memory locations we may have to wait on.
/// Encoded as u16 chunks:
///
@@ -645,6 +660,8 @@ class SIInsertWaitcnts {
WaitcntBrackets &ScoreBrackets);
void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
bool ExpertMode) const;
+ AtomicRMWState getAtomicRMWState(MachineInstr &MI,
+ AtomicRMWState PrevState) const;
};
// This objects maintains the current score brackets of each wait counter, and
@@ -2866,6 +2883,39 @@ void SIInsertWaitcnts::setSchedulingMode(MachineBasicBlock &MBB,
.addImm(EncodedReg);
}
+// Track back-to-back atomic RMW instructions, referred to as a block.
+//
+// Determines whether \p MI starts a new atomic RMW block, is inside
+// an existing block, or is outside of a block. A block is broken when a
+// CU-scoped memory op or an atomic store is encountered. ALU ops
+// and non-memory instructions don't break a block. The function returns
+// the new state after processing the current instruction based on
+// \p PrevState, the previously captured state.
+AtomicRMWState
+SIInsertWaitcnts::getAtomicRMWState(MachineInstr &MI,
+ AtomicRMWState PrevState) const {
+ if (isAtomicRMW(MI)) {
+ // Transition from NotInBlock -> NewBlock -> InsideBlock.
+ if (PrevState == AtomicRMWState::NotInBlock)
+ return AtomicRMWState::NewBlock;
+ if (PrevState == AtomicRMWState::NewBlock)
+ return AtomicRMWState::InsideBlock;
+
+ return PrevState;
+ }
+
+ // LDS memory operations don't break the block.
+ if (TII->isDS(MI) || (TII->isFLAT(MI) && TII->mayAccessLDSThroughFlat(MI)))
+ return PrevState;
+
+ // Reset the atomic RMW block state when found other VMEM and SMEM operations.
+ if (MI.mayLoad() ^ MI.mayStore())
+ return AtomicRMWState::NotInBlock;
+
+ // Return the previous state otherwise.
+ return PrevState;
+}
+
// Generate s_waitcnt instructions where needed.
bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
MachineBasicBlock &Block,
@@ -2894,6 +2944,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
// Walk over the instructions.
MachineInstr *OldWaitcntInstr = nullptr;
+ AtomicRMWState RMWState = AtomicRMWState::NotInBlock;
for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
E = Block.instr_end();
@@ -2903,14 +2954,32 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
++Iter;
continue;
}
+ // Get the atomic RMW block state for current instruction.
+ RMWState = getAtomicRMWState(Inst, RMWState);
// Track pre-existing waitcnts that were added in earlier iterations or by
// the memory legalizer.
if (isWaitInstr(Inst) ||
(IsExpertMode && Inst.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR)) {
- if (!OldWaitcntInstr)
- OldWaitcntInstr = &Inst;
++Iter;
+ bool IsSoftXcnt = isSoftXcnt(Inst);
+ // The Memory Legalizer conservatively inserts a soft xcnt before each
+ // atomic RMW operation. However, for sequences of back-to-back atomic
+ // RMWs, only the first s_wait_xcnt insertion is necessary. Optimize away
+ // the redundant soft xcnts when we're inside an atomic RMW block.
+ if (Iter != E && IsSoftXcnt) {
+ // Check if the next instruction can potentially change the atomic RMW
+ // state.
+ RMWState = getAtomicRMWState(*Iter, RMWState);
+ }
+
+ if (IsSoftXcnt && RMWState == AtomicRMWState::InsideBlock) {
+ // Delete this soft xcnt.
+ Inst.eraseFromParent();
+ Modified = true;
+ } else if (!OldWaitcntInstr) {
+ OldWaitcntInstr = &Inst;
+ }
continue;
}
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
index f699a6b73798f..e3adf737d6e5c 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
@@ -50,7 +50,6 @@ define amdgpu_kernel void @atomic_rmw_back_to_back(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -61,7 +60,6 @@ define amdgpu_kernel void @atomic_rmw_back_to_back(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -104,7 +102,6 @@ define amdgpu_kernel void @atomic_rmw_with_alu(ptr addrspace(1) %ptr, i32 %a, i3
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -116,7 +113,6 @@ define amdgpu_kernel void @atomic_rmw_with_alu(ptr addrspace(1) %ptr, i32 %a, i3
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -159,7 +155,6 @@ define amdgpu_kernel void @atomic_rmw_broken_by_global_load(ptr addrspace(1) %pt
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -212,7 +207,6 @@ define amdgpu_kernel void @atomic_rmw_broken_by_global_store(ptr addrspace(1) %p
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -268,7 +262,6 @@ define amdgpu_kernel void @atomic_rmw_broken_by_flat_load(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -279,7 +272,6 @@ define amdgpu_kernel void @atomic_rmw_broken_by_flat_load(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -321,7 +313,6 @@ define amdgpu_kernel void @atomic_rmw_broken_by_flat_store(ptr addrspace(1) %ptr
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -335,7 +326,6 @@ define amdgpu_kernel void @atomic_rmw_broken_by_flat_store(ptr addrspace(1) %ptr
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -377,7 +367,6 @@ define amdgpu_kernel void @atomic_rmw_broken_by_smem_load(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -432,7 +421,6 @@ define amdgpu_kernel void @atomic_rmw_broken_by_atomic_store(ptr addrspace(1) %p
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -494,7 +482,6 @@ define amdgpu_kernel void @atomic_rmw_with_lds_load(ptr addrspace(1) %ptr, ptr a
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -506,7 +493,6 @@ define amdgpu_kernel void @atomic_rmw_with_lds_load(ptr addrspace(1) %ptr, ptr a
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -548,7 +534,6 @@ define amdgpu_kernel void @atomic_rmw_with_lds_store(ptr addrspace(1) %ptr, ptr
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -562,7 +547,6 @@ define amdgpu_kernel void @atomic_rmw_with_lds_store(ptr addrspace(1) %ptr, ptr
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -604,7 +588,6 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_load(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -626,7 +609,6 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_load(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -669,7 +651,6 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_store(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -694,7 +675,6 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_store(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -740,7 +720,6 @@ define amdgpu_kernel void @atomic_rmw_borken_by_async_lds_copy(ptr addrspace(1)
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -799,7 +778,6 @@ define amdgpu_kernel void @multiple_atomic_rmw_blocks(ptr addrspace(1) %ptr1, pt
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -821,7 +799,6 @@ define amdgpu_kernel void @multiple_atomic_rmw_blocks(ptr addrspace(1) %ptr1, pt
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -864,7 +841,6 @@ define amdgpu_kernel void @different_atomic_ops(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_sub_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -875,7 +851,6 @@ define amdgpu_kernel void @different_atomic_ops(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_and_b32 v0, v1, s[0:1] offset:8 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -886,7 +861,6 @@ define amdgpu_kernel void @different_atomic_ops(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_or_b32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -937,7 +911,6 @@ define amdgpu_kernel void @atomic_rmw_across_basic_blocks(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1031,7 +1004,6 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1095,7 +1067,6 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1248,7 +1219,6 @@ define amdgpu_kernel void @atomic_rmw_fallthrough(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1279,7 +1249,6 @@ define amdgpu_kernel void @atomic_rmw_fallthrough(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
More information about the llvm-commits
mailing list