[llvm] [AMDGPU] Add `isConvergent=1` to `DS_ATOMIC_*_BARRIER` instructions (PR #214166)
Pierre van Houtryve via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 02:15:34 PDT 2026
https://github.com/Pierre-vh created https://github.com/llvm/llvm-project/pull/214166
The intrinsic already has it. The instructions may issue a wake-up
signal on barrier completion so the set of waves that execute the instruction
may not change.
The test case shows taildup duplicating the MachineInstrs when `isConvergent=0`.
Assisted-By: Claude Opus 4.8
Claude found the reproducer for me after I found the lack of `isConvergent=1` on those
instructions.
>From 747abb8167e987963b02698510e99c252e28617f Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Wed, 5 Aug 2026 11:13:13 +0200
Subject: [PATCH] [AMDGPU] Add `isConvergent=1` to `DS_ATOMIC_*_BARRIER`
instructions
The intrinsic already has it. The instructions may issue a wake-up
signal on barrier completion so the set of waves that execute the instruction
may not change.
The test case shows taildup duplicating the MachineInstrs when `isConvergent=0`.
Assisted-By: Claude Opus 4.8
Claude found the reproducer for me after I found the lack of `isConvergent=1` on those
instructions.
---
llvm/lib/Target/AMDGPU/DSInstructions.td | 28 +++--
.../AMDGPU/ds-atomic-barrier-sideeffects.ll | 106 ++++++++++++++++++
2 files changed, 122 insertions(+), 12 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-sideeffects.ll
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 204cd89d4aefb..90713dfc1de7b 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -900,21 +900,25 @@ defm DS_LOAD_TR16_B128 : DS_1A_RET_NoM0<"ds_load_tr16_b128", VGPROp_128>;
} // End WaveSizePredicate = isWave32, mayStore = 0
let OtherPredicates = [HasLdsBarrierArriveAtomic] in {
-let ASYNC_CNT = 1, LGKM_CNT = 0, Uses = [EXEC, ASYNCcnt], Defs = [ASYNCcnt] in {
-def DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 : DS_1A_Off16_NORET<"ds_atomic_async_barrier_arrive_b64">;
-}
+ let isConvergent = 1 in {
+ let ASYNC_CNT = 1, LGKM_CNT = 0, Uses = [EXEC, ASYNCcnt], Defs = [ASYNCcnt] in {
+ def DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 : DS_1A_Off16_NORET<"ds_atomic_async_barrier_arrive_b64">;
+ }
-def : GCNPat <
- (int_amdgcn_ds_atomic_async_barrier_arrive_b64 (DS1Addr1Offset i32:$ptr, i32:$offset)),
- (DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 VGPROp_32:$ptr, Offset:$offset, (i1 0))
->;
+ defm DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 : DS_1A1D_RET_mc_gfx9<"ds_atomic_barrier_arrive_rtn_b64", VGPROp_64>;
+ } // End isConvergent = 1
-defm DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 : DS_1A1D_RET_mc_gfx9<"ds_atomic_barrier_arrive_rtn_b64", VGPROp_64>;
+ def : GCNPat <
+ (int_amdgcn_ds_atomic_async_barrier_arrive_b64 (DS1Addr1Offset i32:$ptr, i32:$offset)),
+ (DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 VGPROp_32:$ptr, Offset:$offset, (i1 0))
+ >;
-def : GCNPat<
- (i64 (int_amdgcn_ds_atomic_barrier_arrive_rtn_b64 (DS1Addr1Offset i32:$ptr, i32:$offset), i64:$data)),
- (DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 $ptr, $data, Offset:$offset, (i1 0))
->;
+
+
+ def : GCNPat<
+ (i64 (int_amdgcn_ds_atomic_barrier_arrive_rtn_b64 (DS1Addr1Offset i32:$ptr, i32:$offset), i64:$data)),
+ (DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 $ptr, $data, Offset:$offset, (i1 0))
+ >;
} // End OtherPredicates = [HasLdsBarrierArriveAtomic]
} // End SubtargetPredicate = isGFX1250Plus
diff --git a/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-sideeffects.ll b/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-sideeffects.ll
new file mode 100644
index 0000000000000..d55944417410c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-atomic-barrier-sideeffects.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu12.50 -O2 -tail-dup-size=1000 -tail-dup-placement-threshold=1000 -enable-tail-merge=0 < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+
+; llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64 does more than a plain atomic,
+; it may issue a wakeup to other waves. It should not be duplicated.
+
+declare i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3), i64)
+
+; test using taildup
+define void @taildup_ds_atomic_barrier_arrive(ptr addrspace(1) %a, ptr addrspace(3) %bar, i64 %data, i1 %cond) {
+; GCN-LABEL: taildup_ds_atomic_barrier_arrive:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v6, v3
+; GCN-NEXT: v_and_b32_e32 v3, 1, v5
+; GCN-NEXT: s_mov_b32 s0, exec_lo
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT: v_cmpx_ne_u32_e32 1, v3
+; GCN-NEXT: s_xor_b32 s0, exec_lo, s0
+; GCN-NEXT: s_cbranch_execz .LBB0_2
+; GCN-NEXT: ; %bb.1: ; %bb2
+; GCN-NEXT: v_mov_b32_e32 v3, 1
+; GCN-NEXT: global_store_b32 v[0:1], v3, off
+; GCN-NEXT: .LBB0_2: ; %Flow
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GCN-NEXT: s_cbranch_execz .LBB0_4
+; GCN-NEXT: ; %bb.3: ; %bb1
+; GCN-NEXT: v_mov_b32_e32 v3, 0
+; GCN-NEXT: global_store_b32 v[0:1], v3, off
+; GCN-NEXT: .LBB0_4: ; %call
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GCN-NEXT: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[6:7]
+; GCN-NEXT: s_wait_dscnt 0x0
+; GCN-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ br i1 %cond, label %bb1, label %bb2
+
+bb1:
+ store i32 0, ptr addrspace(1) %a
+ br label %call
+
+bb2:
+ store i32 1, ptr addrspace(1) %a
+ br label %call
+
+call:
+ %r = call i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3) %bar, i64 %data)
+ store i64 %r, ptr addrspace(1) %a
+ br label %ret
+
+ret:
+ ret void
+}
+
+; async test using taildup
+define void @taildup_ds_atomic_async_barrier_arrive(ptr addrspace(1) %a, ptr addrspace(3) %bar, i64 %data, i1 %cond) {
+; GCN-LABEL: taildup_ds_atomic_async_barrier_arrive:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: v_and_b32_e32 v3, 1, v5
+; GCN-NEXT: s_mov_b32 s0, exec_lo
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT: v_cmpx_ne_u32_e32 1, v3
+; GCN-NEXT: s_xor_b32 s0, exec_lo, s0
+; GCN-NEXT: s_cbranch_execz .LBB1_2
+; GCN-NEXT: ; %bb.1: ; %bb2
+; GCN-NEXT: v_mov_b32_e32 v3, 1
+; GCN-NEXT: global_store_b32 v[0:1], v3, off
+; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN-NEXT: .LBB1_2: ; %Flow
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GCN-NEXT: s_cbranch_execz .LBB1_4
+; GCN-NEXT: ; %bb.3: ; %bb1
+; GCN-NEXT: v_mov_b32_e32 v3, 0
+; GCN-NEXT: global_store_b32 v[0:1], v3, off
+; GCN-NEXT: .LBB1_4: ; %call
+; GCN-NEXT: s_wait_xcnt 0x0
+; GCN-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT: ds_atomic_async_barrier_arrive_b64 v2
+; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
+; GCN-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ br i1 %cond, label %bb1, label %bb2
+
+bb1:
+ store i32 0, ptr addrspace(1) %a
+ br label %call
+
+bb2:
+ store i32 1, ptr addrspace(1) %a
+ br label %call
+
+call:
+ call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
+ br label %ret
+
+ret:
+ ret void
+}
More information about the llvm-commits
mailing list