[llvm] [AMDGPU] Fix chain handling when lowering barrier intrinsics (PR #109799)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 24 06:29:27 PDT 2024
https://github.com/jayfoad created https://github.com/llvm/llvm-project/pull/109799
Previously we would fail an assertion in RemoveNodeFromCSEMaps after
lowering:
t3: ch = llvm.amdgcn.s.barrier.join t0, TargetConstant:i64<2973>, Constant:i32<0>
to:
t6: ch = S_BARRIER_JOIN_IMM TargetConstant:i32<0>
>From 8ec9ffe1792679c8e72a9e62731c1321ceac05dc Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Tue, 24 Sep 2024 14:10:44 +0100
Subject: [PATCH] [AMDGPU] Fix chain handling when lowering barrier intrinsics
Previously we would fail an assertion in RemoveNodeFromCSEMaps after
lowering:
t3: ch = llvm.amdgcn.s.barrier.join t0, TargetConstant:i64<2973>, Constant:i32<0>
to:
t6: ch = S_BARRIER_JOIN_IMM TargetConstant:i32<0>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
.../AMDGPU/llvm.amdgcn.s.barrier.wait.ll | 74 +++++++++++++------
2 files changed, 53 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a9754ba357893f..2aa963c2359a74 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -9967,7 +9967,9 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
0);
}
Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
- } else if (!IsInlinableBarID) {
+ } else if (IsInlinableBarID) {
+ Ops.push_back(Chain);
+ } else {
Ops.push_back(copyToM0(DAG, Chain, DL, BarOp).getValue(0));
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.wait.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.wait.ll
index a4be9ed8c2b4af..fe4e87c07dd260 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.wait.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.wait.ll
@@ -768,17 +768,19 @@ define void @test5_s_barrier_init_m0(i32 %arg1 ,i32 %arg2) {
}
define amdgpu_kernel void @test1_s_barrier_join(ptr addrspace(1) %out) #0 {
+;
; GFX12-SDAG-LABEL: test1_s_barrier_join:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT: s_barrier_join -1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
; GFX12-SDAG-NEXT: v_mul_u32_u24_e32 v1, v0, v0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX12-SDAG-NEXT: v_sub_nc_u32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-SDAG-NEXT: s_barrier_join -1
+; GFX12-SDAG-NEXT: global_store_b32 v3, v0, s[0:1]
; GFX12-SDAG-NEXT: s_nop 0
; GFX12-SDAG-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX12-SDAG-NEXT: s_endpgm
@@ -810,17 +812,19 @@ entry:
}
define amdgpu_kernel void @test2_s_barrier_join(ptr addrspace(1) %out) #0 {
+;
; GFX12-SDAG-LABEL: test2_s_barrier_join:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT: s_barrier_join 1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
; GFX12-SDAG-NEXT: v_mul_u32_u24_e32 v1, v0, v0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX12-SDAG-NEXT: v_sub_nc_u32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-SDAG-NEXT: s_barrier_join 1
+; GFX12-SDAG-NEXT: global_store_b32 v3, v0, s[0:1]
; GFX12-SDAG-NEXT: s_nop 0
; GFX12-SDAG-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX12-SDAG-NEXT: s_endpgm
@@ -852,17 +856,19 @@ entry:
}
define amdgpu_kernel void @test3_s_barrier_join(ptr addrspace(1) %out) #0 {
+;
; GFX12-SDAG-LABEL: test3_s_barrier_join:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT: s_barrier_join 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
; GFX12-SDAG-NEXT: v_mul_u32_u24_e32 v1, v0, v0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX12-SDAG-NEXT: v_sub_nc_u32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-SDAG-NEXT: s_barrier_join 0
+; GFX12-SDAG-NEXT: global_store_b32 v3, v0, s[0:1]
; GFX12-SDAG-NEXT: s_nop 0
; GFX12-SDAG-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX12-SDAG-NEXT: s_endpgm
@@ -967,6 +973,20 @@ define void @test5_s_barrier_join_m0(i32 %arg) {
ret void
}
+define void @test6_s_barrier_join_0() {
+; GFX12-LABEL: test6_s_barrier_join_0:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_barrier_join 0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.s.barrier.join(i32 0)
+ ret void
+}
+
define amdgpu_kernel void @test1_s_barrier_leave(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %out) #0 {
; GFX12-SDAG-LABEL: test1_s_barrier_leave:
; GFX12-SDAG: ; %bb.0: ; %entry
@@ -1026,17 +1046,19 @@ entry:
}
define amdgpu_kernel void @test1_s_wakeup_barrier(ptr addrspace(1) %out) #0 {
+;
; GFX12-SDAG-LABEL: test1_s_wakeup_barrier:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT: s_wakeup_barrier -1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
; GFX12-SDAG-NEXT: v_mul_u32_u24_e32 v1, v0, v0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX12-SDAG-NEXT: v_sub_nc_u32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-SDAG-NEXT: s_wakeup_barrier -1
+; GFX12-SDAG-NEXT: global_store_b32 v3, v0, s[0:1]
; GFX12-SDAG-NEXT: s_nop 0
; GFX12-SDAG-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX12-SDAG-NEXT: s_endpgm
@@ -1068,17 +1090,19 @@ entry:
}
define amdgpu_kernel void @test2_s_wakeup_barrier(ptr addrspace(1) %out) #0 {
+;
; GFX12-SDAG-LABEL: test2_s_wakeup_barrier:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT: s_wakeup_barrier 1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
; GFX12-SDAG-NEXT: v_mul_u32_u24_e32 v1, v0, v0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX12-SDAG-NEXT: v_sub_nc_u32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-SDAG-NEXT: s_wakeup_barrier 1
+; GFX12-SDAG-NEXT: global_store_b32 v3, v0, s[0:1]
; GFX12-SDAG-NEXT: s_nop 0
; GFX12-SDAG-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX12-SDAG-NEXT: s_endpgm
@@ -1110,17 +1134,19 @@ entry:
}
define amdgpu_kernel void @test3_s_wakeup_barrier(ptr addrspace(1) %out) #0 {
+;
; GFX12-SDAG-LABEL: test3_s_wakeup_barrier:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT: s_wakeup_barrier 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
; GFX12-SDAG-NEXT: v_mul_u32_u24_e32 v1, v0, v0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX12-SDAG-NEXT: v_sub_nc_u32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-SDAG-NEXT: s_wakeup_barrier 0
+; GFX12-SDAG-NEXT: global_store_b32 v3, v0, s[0:1]
; GFX12-SDAG-NEXT: s_nop 0
; GFX12-SDAG-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX12-SDAG-NEXT: s_endpgm
More information about the llvm-commits
mailing list