[llvm] [AMDGPU] Preserve metadata in all barrier lowering paths (PR #191916)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 17:49:36 PDT 2026
https://github.com/hidekisaito created https://github.com/llvm/llvm-project/pull/191916
Extend copyMetadata to every call-to-call replacement in
AMDGPULowerIntrinsics, not just the single-wave s_barrier →
wave_barrier path. This covers:
- s_cluster_barrier → wave_barrier (single-wave)
- s_cluster_barrier → signal_isfirst + wait + signal + wait (multi-wave)
- s_barrier → signal + wait (split barriers)
Add GFX11 and GFX12 RUN lines and test functions for all lowering
paths to verify metadata preservation.
Made-with: Cursor
>From 00680c97ec467a4444cb53a4570658f8698bf393 Mon Sep 17 00:00:00 2001
From: Hideki Saito <hidekido at amd.com>
Date: Mon, 13 Apr 2026 19:41:39 -0500
Subject: [PATCH] [AMDGPU] Preserve metadata in all barrier lowering paths
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Extend copyMetadata to every call-to-call replacement in
AMDGPULowerIntrinsics, not just the single-wave s_barrier →
wave_barrier path. This covers:
- s_cluster_barrier → wave_barrier (single-wave)
- s_cluster_barrier → signal_isfirst + wait + signal + wait (multi-wave)
- s_barrier → signal + wait (split barriers)
Add GFX11 and GFX12 RUN lines and test functions for all lowering
paths to verify metadata preservation.
Made-with: Cursor
---
.../Target/AMDGPU/AMDGPULowerIntrinsics.cpp | 18 +-
.../lower-intrinsics-noalias-metadata.ll | 166 ++++++++++++++++--
2 files changed, 164 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
index b158ceae3a3c1..d9b018e3cefa0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
@@ -110,11 +110,13 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
} else {
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::WORKGROUP);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::WORKGROUP);
- Value *IsFirst = B.CreateIntrinsic(
+ CallInst *IsFirst = B.CreateIntrinsic(
B.getInt1Ty(), Intrinsic::amdgcn_s_barrier_signal_isfirst,
{BarrierID_32});
+ IsFirst->copyMetadata(I);
B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16});
+ {BarrierID_16})
+ ->copyMetadata(I);
Instruction *ThenTerm =
SplitBlockAndInsertIfThen(IsFirst, I.getIterator(), false);
@@ -126,11 +128,13 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::CLUSTER);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::CLUSTER);
B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_signal,
- {BarrierID_32});
+ {BarrierID_32})
+ ->copyMetadata(I);
B.SetInsertPoint(&I);
B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16});
+ {BarrierID_16})
+ ->copyMetadata(I);
I.eraseFromParent();
return true;
@@ -173,9 +177,11 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::WORKGROUP);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::WORKGROUP);
B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_signal,
- {BarrierID_32});
+ {BarrierID_32})
+ ->copyMetadata(I);
B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16});
+ {BarrierID_16})
+ ->copyMetadata(I);
I.eraseFromParent();
return true;
}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-intrinsics-noalias-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-intrinsics-noalias-metadata.ll
index e4d09d83830a9..7a106e9f18ef8 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-intrinsics-noalias-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-intrinsics-noalias-metadata.ll
@@ -1,21 +1,42 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes='amdgpu-lower-intrinsics,function(amdgpu-annotate-uniform)' -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -codegen-opt-level=1 -mattr=+wavefrontsize32 < %s | FileCheck %s
+; RUN: opt -passes='amdgpu-lower-intrinsics,function(amdgpu-annotate-uniform)' -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -codegen-opt-level=1 -mattr=+wavefrontsize32 < %s | FileCheck --check-prefix=NOCLOBBER %s
+; RUN: opt -passes=amdgpu-lower-intrinsics -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -codegen-opt-level=1 -mattr=+wavefrontsize32 < %s | FileCheck --check-prefix=GFX11 %s
+; RUN: opt -passes=amdgpu-lower-intrinsics -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -codegen-opt-level=1 -mattr=+wavefrontsize32 < %s | FileCheck --check-prefix=GFX12 %s
-; When amdgpu-lower-intrinsics downgrades s_barrier to wave_barrier on a
-; single-wave workgroup it must preserve scoped-AA metadata (!noalias,
-; !alias.scope, !tbaa). Without it MemorySSA can no longer see past the
-; barrier, walks further, and may reach an unrelated side-effecting def
-; that falsely blocks !amdgpu.noclobber on a later global load.
+; Verify that amdgpu-lower-intrinsics preserves metadata when lowering
+; barrier intrinsics to other barrier intrinsics or wave_barrier.
+; The original noclobber motivation test: without metadata preservation,
+; MemorySSA can no longer see past the barrier, walks further, and may
+; reach an unrelated side-effecting def that falsely blocks
+; !amdgpu.noclobber on a later global load.
define amdgpu_kernel void @noclobber_after_barrier_lowering(ptr addrspace(1) %arg, ptr addrspace(3) %lds) "amdgpu-flat-work-group-size"="1,32" {
-; CHECK-LABEL: @noclobber_after_barrier_lowering(
-; CHECK-NEXT: bb:
-; CHECK-NEXT: store i32 99, ptr addrspace(3) [[LDS:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
-; CHECK-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG:%.*]], i64 1, !amdgpu.uniform [[META9:![0-9]+]]
-; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4, !alias.scope [[META7]], !noalias [[META4]], !amdgpu.noclobber [[META9]]
-; CHECK-NEXT: store i32 [[VAL]], ptr addrspace(1) [[ARG]], align 4
-; CHECK-NEXT: ret void
+; NOCLOBBER-LABEL: @noclobber_after_barrier_lowering(
+; NOCLOBBER-NEXT: bb:
+; NOCLOBBER-NEXT: store i32 99, ptr addrspace(3) [[LDS:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
+; NOCLOBBER-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG:%.*]], i64 1, !amdgpu.uniform [[META9:![0-9]+]]
+; NOCLOBBER-NEXT: [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4, !alias.scope [[META7]], !noalias [[META4]], !amdgpu.noclobber [[META9]]
+; NOCLOBBER-NEXT: store i32 [[VAL]], ptr addrspace(1) [[ARG]], align 4
+; NOCLOBBER-NEXT: ret void
+;
+; GFX11-LABEL: @noclobber_after_barrier_lowering(
+; GFX11-NEXT: bb:
+; GFX11-NEXT: store i32 99, ptr addrspace(3) [[LDS:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
+; GFX11-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG:%.*]], i64 1
+; GFX11-NEXT: [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4, !alias.scope [[META7]], !noalias [[META4]]
+; GFX11-NEXT: store i32 [[VAL]], ptr addrspace(1) [[ARG]], align 4
+; GFX11-NEXT: ret void
+;
+; GFX12-LABEL: @noclobber_after_barrier_lowering(
+; GFX12-NEXT: bb:
+; GFX12-NEXT: store i32 99, ptr addrspace(3) [[LDS:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
+; GFX12-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG:%.*]], i64 1
+; GFX12-NEXT: [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4, !alias.scope [[META7]], !noalias [[META4]]
+; GFX12-NEXT: store i32 [[VAL]], ptr addrspace(1) [[ARG]], align 4
+; GFX12-NEXT: ret void
;
bb:
store i32 99, ptr addrspace(3) %lds, align 4, !alias.scope !4, !noalias !1, !tbaa !5
@@ -26,7 +47,124 @@ bb:
ret void
}
+; Single-wave s_barrier -> wave_barrier
+define amdgpu_kernel void @barrier_single_wave(ptr addrspace(1) %arg) "amdgpu-flat-work-group-size"="1,32" {
+; NOCLOBBER-LABEL: @barrier_single_wave(
+; NOCLOBBER-NEXT: bb:
+; NOCLOBBER-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: ret void
+;
+; GFX11-LABEL: @barrier_single_wave(
+; GFX11-NEXT: bb:
+; GFX11-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: ret void
+;
+; GFX12-LABEL: @barrier_single_wave(
+; GFX12-NEXT: bb:
+; GFX12-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: ret void
+;
+bb:
+ tail call void @llvm.amdgcn.s.barrier(), !noalias !1, !alias.scope !4, !tbaa !5
+ ret void
+}
+
+; Multi-wave s_barrier -> s_barrier_signal + s_barrier_wait (GFX12 split barriers)
+define amdgpu_kernel void @barrier_multi_wave(ptr addrspace(1) %arg) {
+; NOCLOBBER-LABEL: @barrier_multi_wave(
+; NOCLOBBER-NEXT: bb:
+; NOCLOBBER-NEXT: tail call void @llvm.amdgcn.s.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: ret void
+;
+; GFX11-LABEL: @barrier_multi_wave(
+; GFX11-NEXT: bb:
+; GFX11-NEXT: tail call void @llvm.amdgcn.s.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: ret void
+;
+; GFX12-LABEL: @barrier_multi_wave(
+; GFX12-NEXT: bb:
+; GFX12-NEXT: call void @llvm.amdgcn.s.barrier.signal(i32 -1), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -1), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: ret void
+;
+bb:
+ tail call void @llvm.amdgcn.s.barrier(), !noalias !1, !alias.scope !4, !tbaa !5
+ ret void
+}
+
+; Single-wave s_cluster_barrier -> wave_barrier
+define amdgpu_kernel void @cluster_barrier_single_wave(ptr addrspace(1) %arg) "amdgpu-flat-work-group-size"="1,32" {
+; NOCLOBBER-LABEL: @cluster_barrier_single_wave(
+; NOCLOBBER-NEXT: bb:
+; NOCLOBBER-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: call void @llvm.amdgcn.s.barrier.signal(i32 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: ret void
+;
+; GFX11-LABEL: @cluster_barrier_single_wave(
+; GFX11-NEXT: bb:
+; GFX11-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: call void @llvm.amdgcn.s.barrier.signal(i32 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: ret void
+;
+; GFX12-LABEL: @cluster_barrier_single_wave(
+; GFX12-NEXT: bb:
+; GFX12-NEXT: call void @llvm.amdgcn.wave.barrier(), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: call void @llvm.amdgcn.s.barrier.signal(i32 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: ret void
+;
+bb:
+ tail call void @llvm.amdgcn.s.cluster.barrier(), !noalias !1, !alias.scope !4, !tbaa !5
+ ret void
+}
+
+; Multi-wave s_cluster_barrier -> signal_isfirst + wait + signal(cluster) + wait(cluster)
+define amdgpu_kernel void @cluster_barrier_multi_wave(ptr addrspace(1) %arg) {
+; NOCLOBBER-LABEL: @cluster_barrier_multi_wave(
+; NOCLOBBER-NEXT: bb:
+; NOCLOBBER-NEXT: [[TMP0:%.*]] = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 -1), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -1), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: br i1 [[TMP0]], label [[TMP1:%.*]], label [[TMP2:%.*]], !amdgpu.uniform [[META9]]
+; NOCLOBBER: 1:
+; NOCLOBBER-NEXT: call void @llvm.amdgcn.s.barrier.signal(i32 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: br label [[TMP2]]
+; NOCLOBBER: 2:
+; NOCLOBBER-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; NOCLOBBER-NEXT: ret void
+;
+; GFX11-LABEL: @cluster_barrier_multi_wave(
+; GFX11-NEXT: bb:
+; GFX11-NEXT: [[TMP0:%.*]] = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 -1), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -1), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: br i1 [[TMP0]], label [[TMP1:%.*]], label [[TMP2:%.*]]
+; GFX11: 1:
+; GFX11-NEXT: call void @llvm.amdgcn.s.barrier.signal(i32 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: br label [[TMP2]]
+; GFX11: 2:
+; GFX11-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX11-NEXT: ret void
+;
+; GFX12-LABEL: @cluster_barrier_multi_wave(
+; GFX12-NEXT: bb:
+; GFX12-NEXT: [[TMP0:%.*]] = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 -1), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -1), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: br i1 [[TMP0]], label [[TMP1:%.*]], label [[TMP2:%.*]]
+; GFX12: 1:
+; GFX12-NEXT: call void @llvm.amdgcn.s.barrier.signal(i32 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: br label [[TMP2]]
+; GFX12: 2:
+; GFX12-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -3), !tbaa [[TBAA0]], !alias.scope [[META4]], !noalias [[META7]]
+; GFX12-NEXT: ret void
+;
+bb:
+ tail call void @llvm.amdgcn.s.cluster.barrier(), !noalias !1, !alias.scope !4, !tbaa !5
+ ret void
+}
+
declare void @llvm.amdgcn.s.barrier()
+declare void @llvm.amdgcn.s.cluster.barrier()
!0 = distinct !{!0, !"domain"}
!2 = distinct !{!2, !0, !"global_scope"}
More information about the llvm-commits
mailing list