[llvm] [AMDGPU] Add object linking support for LDS and named barrier lowering in the middle end (PR #191645)
Shilei Tian via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 06:31:33 PDT 2026
https://github.com/shiltian updated https://github.com/llvm/llvm-project/pull/191645
>From 7a7fb028ceb667198060223ba41813b96d643574 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i at tianshilei.me>
Date: Sat, 11 Apr 2026 12:29:13 -0400
Subject: [PATCH] [AMDGPU] Add object linking support for LDS and named barrier
lowering in the middle end
This is the first patch in a series introducing object linking support for
AMDGPU.
This PR adds the -amdgpu-enable-object-linking flag to enable object linking in
the backend. It also updates the AMDGPULowerModuleLDSPass and
AMDGPULowerExecSync passes to support lowering LDS and named barrier globals
when object linking is enabled.
---
.../lib/Target/AMDGPU/AMDGPULowerExecSync.cpp | 44 +++++
.../AMDGPU/AMDGPULowerModuleLDSPass.cpp | 167 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 7 +
llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h | 1 +
.../AMDGPU/lds-link-time-named-barrier.ll | 35 ++++
.../lower-module-lds-link-time-classify.ll | 73 ++++++++
...lower-module-lds-link-time-global-scope.ll | 125 +++++++++++++
...ower-module-lds-link-time-internal-func.ll | 38 ++++
...odule-lds-link-time-internal-multi-user.ll | 50 ++++++
9 files changed, 540 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-classify.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-global-scope.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-func.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-multi-user.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
index c26e97360efef..58f79d8fe059b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
@@ -26,6 +26,7 @@
#include "llvm/IR/ReplaceConstant.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <algorithm>
@@ -178,7 +179,50 @@ static bool lowerExecSyncGlobalVariables(
return Changed;
}
+// With object linking, barrier ID assignment is deferred to the linker.
+// Externalize named barrier globals and emit self-contained metadata so the
+// AsmPrinter can generate the callgraph entries the linker needs.
+static bool handleNamedBarriersForObjectLinking(Module &M) {
+ DenseMap<GlobalVariable *, DenseSet<Function *>> BarrierToFuncs;
+ for (GlobalVariable &GV : M.globals()) {
+ if (!isNamedBarrier(GV) || GV.use_empty())
+ continue;
+ for (User *U : GV.users()) {
+ if (auto *I = dyn_cast<Instruction>(U))
+ BarrierToFuncs[&GV].insert(I->getFunction());
+ }
+ }
+ if (BarrierToFuncs.empty())
+ return false;
+
+ LLVMContext &Ctx = M.getContext();
+ NamedMDNode *BarMD = M.getOrInsertNamedMetadata("amdgpu.named_barrier.uses");
+
+ std::string ModuleId;
+ ModuleId = getUniqueModuleId(&M);
+ assert(!ModuleId.empty() &&
+ "modules with named barriers should have a unique ID");
+ for (auto &[V, Funcs] : BarrierToFuncs) {
+ if (V->hasLocalLinkage())
+ V->setName("__amdgpu_named_barrier." + V->getName() + ModuleId);
+ else if (!V->getName().starts_with("__amdgpu_named_barrier"))
+ V->setName("__amdgpu_named_barrier." + V->getName());
+ V->setInitializer(nullptr);
+ V->setLinkage(GlobalValue::ExternalLinkage);
+
+ SmallVector<Metadata *, 4> Ops;
+ Ops.push_back(ValueAsMetadata::get(V));
+ for (Function *F : Funcs)
+ Ops.push_back(ValueAsMetadata::get(F));
+ BarMD->addOperand(MDNode::get(Ctx, Ops));
+ }
+ return true;
+}
+
static bool runLowerExecSyncGlobals(Module &M) {
+ if (AMDGPUTargetMachine::EnableObjectLinking)
+ return handleNamedBarriersForObjectLinking(M);
+
CallGraph CG = CallGraph(M);
bool Changed = false;
Changed |= eliminateConstantExprUsesOfLDSFromAllInstructions(M);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index 699cf7d4d1cdd..ebb8b9d4afbef 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -185,6 +185,7 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SmallString.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -910,7 +911,173 @@ class AMDGPULowerModuleLDS {
return KernelToCreatedDynamicLDS;
}
+ // Per-TU mode for link-time LDS resolution. Instead of computing a global
+ // layout, create per-function LDS struct declarations so the linker can
+ // assign offsets across TUs.
+ bool runOnModuleLinkTime(Module &M) {
+ bool Changed = superAlignLDSGlobals(M);
+ Changed |= eliminateConstantExprUsesOfLDSFromAllInstructions(M);
+
+ CallGraph CG(M);
+ FunctionVariableMap KernelLDSUses, FunctionLDSUses;
+ getUsesOfLDSByFunction(CG, M, KernelLDSUses, FunctionLDSUses);
+
+ if (KernelLDSUses.empty() && FunctionLDSUses.empty())
+ return Changed;
+
+ std::string ModuleId = getUniqueModuleId(&M);
+ assert(!ModuleId.empty() &&
+ "modules with LDS variables should have a unique ID");
+
+ FunctionVariableMap AllLDSUses;
+ for (auto &[F, Vars] : KernelLDSUses)
+ AllLDSUses[F].insert(Vars.begin(), Vars.end());
+ for (auto &[F, Vars] : FunctionLDSUses)
+ AllLDSUses[F].insert(Vars.begin(), Vars.end());
+
+ // Named barriers are handled by AMDGPULowerExecSync; filter them out.
+ for (auto &[F, Vars] : AllLDSUses) {
+ SmallVector<GlobalVariable *> Barriers;
+ for (GlobalVariable *V : Vars)
+ if (AMDGPU::isNamedBarrier(*V))
+ Barriers.push_back(V);
+ for (GlobalVariable *V : Barriers)
+ Vars.erase(V);
+ }
+
+ // Build reverse map: LDS variable -> functions that use it.
+ DenseMap<GlobalVariable *, SmallVector<Function *, 4>> VarToFuncs;
+ for (auto &[F, Vars] : AllLDSUses) {
+ for (GlobalVariable *V : Vars)
+ VarToFuncs[V].push_back(F);
+ }
+
+ // A variable is function-scope iff it has local linkage and exactly one
+ // user function. Everything else is global-scope and must remain as a
+ // standalone external declaration so the linker can assign a single shared
+ // offset.
+ DenseSet<GlobalVariable *> GlobalScopeVars;
+ DenseSet<GlobalVariable *> InternalMultiUserVars;
+ for (auto &[V, Funcs] : VarToFuncs) {
+ if (!V->hasLocalLinkage() || Funcs.size() > 1) {
+ GlobalScopeVars.insert(V);
+ if (V->hasLocalLinkage())
+ InternalMultiUserVars.insert(V);
+ }
+ }
+
+ // Wrap function-scope LDS into per-function structs (unchanged logic,
+ // but global-scope variables are excluded from the set).
+ SmallVector<std::pair<Function *, GlobalVariable *>, 4> FuncToLdsStruct;
+ DenseSet<GlobalVariable *> AllReplacedVars;
+ for (auto &KV : AllLDSUses) {
+ Function *F = KV.first;
+ DenseSet<GlobalVariable *> FuncScopeVars;
+ for (GlobalVariable *V : KV.second) {
+ if (!GlobalScopeVars.count(V))
+ FuncScopeVars.insert(V);
+ }
+
+ if (FuncScopeVars.empty())
+ continue;
+
+ std::string StructName =
+ F->hasLocalLinkage()
+ ? ("__amdgpu_lds." + F->getName() + ModuleId).str()
+ : ("__amdgpu_lds." + F->getName()).str();
+ LDSVariableReplacement Replacement =
+ createLDSVariableReplacement(M, StructName, FuncScopeVars);
+
+ GlobalVariable *SGV = Replacement.SGV;
+ SGV->setLinkage(GlobalValue::ExternalLinkage);
+ SGV->setInitializer(nullptr);
+ FuncToLdsStruct.push_back({F, SGV});
+
+ replaceLDSVariablesWithStruct(
+ M, FuncScopeVars, Replacement, [F](const Use &U) {
+ auto *I = dyn_cast<Instruction>(U.getUser());
+ return I && I->getFunction() == F;
+ });
+
+ AllReplacedVars.insert(FuncScopeVars.begin(), FuncScopeVars.end());
+ }
+
+ // Internal-linkage LDS variables used by multiple functions would collide
+ // across TUs if promoted individually to external linkage (same name in
+ // different TUs). Pack them into a single per-module struct with a
+ // module-unique name so the linker treats them as one allocation unit.
+ if (!InternalMultiUserVars.empty()) {
+ std::string StructName = "__amdgpu_lds.__internal" + ModuleId;
+ LDSVariableReplacement Replacement =
+ createLDSVariableReplacement(M, StructName, InternalMultiUserVars);
+
+ GlobalVariable *SGV = Replacement.SGV;
+ SGV->setLinkage(GlobalValue::ExternalLinkage);
+ SGV->setInitializer(nullptr);
+
+ replaceLDSVariablesWithStruct(
+ M, InternalMultiUserVars, Replacement,
+ [](const Use &U) { return isa<Instruction>(U.getUser()); });
+
+ DenseSet<Function *> FuncsUsingInternalVars;
+ for (GlobalVariable *V : InternalMultiUserVars) {
+ for (Function *F : VarToFuncs[V])
+ FuncsUsingInternalVars.insert(F);
+ }
+ for (Function *F : FuncsUsingInternalVars)
+ FuncToLdsStruct.push_back({F, SGV});
+
+ AllReplacedVars.insert(InternalMultiUserVars.begin(),
+ InternalMultiUserVars.end());
+ }
+
+ // Convert global-scope LDS to external declarations. Their uses remain
+ // intact and ISel generates R_AMDGPU_ABS32_LO relocations for them.
+ for (GlobalVariable *V : GlobalScopeVars) {
+ V->setInitializer(nullptr);
+ V->setLinkage(GlobalValue::ExternalLinkage);
+ }
+
+ // Emit amdgpu.lds.uses metadata for struct and global-scope LDS.
+ {
+ LLVMContext &Ctx = M.getContext();
+ NamedMDNode *LdsMD = M.getOrInsertNamedMetadata("amdgpu.lds.uses");
+
+ for (auto &[F, SGV] : FuncToLdsStruct)
+ LdsMD->addOperand(MDNode::get(
+ Ctx, {ValueAsMetadata::get(F), ValueAsMetadata::get(SGV)}));
+
+ for (auto &[V, Funcs] : VarToFuncs) {
+ if (GlobalScopeVars.count(V) && !InternalMultiUserVars.count(V)) {
+ for (Function *F : Funcs) {
+ LdsMD->addOperand(MDNode::get(
+ Ctx, {ValueAsMetadata::get(F), ValueAsMetadata::get(V)}));
+ }
+ }
+ }
+ }
+
+ M.addModuleFlag(Module::Error, "amdgpu-link-time-lds", 1);
+
+ DenseSet<GlobalVariable *> AllLDSVarsForCleanup = AllReplacedVars;
+ AllLDSVarsForCleanup.insert(GlobalScopeVars.begin(), GlobalScopeVars.end());
+ removeLocalVarsFromUsedLists(M, AllLDSVarsForCleanup);
+ for (GlobalVariable *GV : AllReplacedVars) {
+ GV->removeDeadConstantUsers();
+ if (GV->use_empty())
+ GV->eraseFromParent();
+ }
+
+ return true;
+ }
+
bool runOnModule(Module &M) {
+ if (AMDGPUTargetMachine::EnableObjectLinking)
+ return runOnModuleLinkTime(M);
+ return runOnModuleNormal(M);
+ }
+
+ bool runOnModuleNormal(Module &M) {
CallGraph CG = CallGraph(M);
bool Changed = superAlignLDSGlobals(M);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index f0d3e7056f69e..e08345f5acbc5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -546,6 +546,12 @@ static cl::opt<bool>
"and asan instrument resulting IR."),
cl::init(true), cl::Hidden);
+static cl::opt<bool, true> EnableObjectLinking(
+ "amdgpu-enable-object-linking",
+ cl::desc("Enable object linking for cross-TU LDS and ABI support"),
+ cl::location(AMDGPUTargetMachine::EnableObjectLinking), cl::init(false),
+ cl::Hidden);
+
static cl::opt<bool, true> EnableLowerModuleLDS(
"amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),
cl::location(AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(true),
@@ -877,6 +883,7 @@ AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
}
bool AMDGPUTargetMachine::EnableFunctionCalls = false;
+bool AMDGPUTargetMachine::EnableObjectLinking = false;
bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;
AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 1be3e5f98ebb8..e2c27f3822380 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -40,6 +40,7 @@ class AMDGPUTargetMachine : public CodeGenTargetMachineImpl {
public:
static bool EnableFunctionCalls;
+ static bool EnableObjectLinking;
static bool EnableLowerModuleLDS;
AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
new file mode 100644
index 0000000000000..e1ad6dcd0b6a3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-enable-object-linking -passes=amdgpu-lower-exec-sync,amdgpu-lower-module-lds < %s | FileCheck %s
+
+; Verify that with object linking enabled:
+; 1. AMDGPULowerExecSync externalizes named barriers and emits
+; amdgpu.named_barrier.uses metadata with (barrier, func...) format
+; 2. AMDGPULowerModuleLDS does not handle named barriers at all
+; 3. amdgpu.lds.uses does NOT contain barrier entries
+
+ at bar = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
+ at lds = internal addrspace(3) global [4 x i32] poison, align 4
+
+; Internal named barrier becomes external with a module-unique hash suffix.
+; CHECK: @[[BAR:__amdgpu_named_barrier\.bar\.[a-f0-9]+]] = external dso_local addrspace(3) global target("amdgcn.named.barrier", 0)
+; CHECK-NOT: !absolute_symbol
+; Regular LDS is packed into the per-function struct (external, for linker).
+; CHECK: @__amdgpu_lds.kernel = external dso_local addrspace(3) global %__amdgpu_lds.kernel.t, align 16
+
+define amdgpu_kernel void @kernel(i32 %idx) {
+; CHECK-LABEL: define amdgpu_kernel void @kernel(
+; CHECK: call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @[[BAR]], i32 3)
+; CHECK: call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @[[BAR]])
+ call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar, i32 3)
+ call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar)
+ call void @llvm.amdgcn.s.barrier.wait(i16 1)
+ %gep = getelementptr [4 x i32], ptr addrspace(3) @lds, i32 0, i32 %idx
+ store i32 42, ptr addrspace(3) %gep, align 4
+ ret void
+}
+
+; Named barrier metadata: (barrier_sym, func1, ...) -- emitted by ExecSync.
+; CHECK-DAG: !amdgpu.named_barrier.uses = !{[[BAR_MD:![0-9]+]]}
+; CHECK-DAG: [[BAR_MD]] = !{ptr addrspace(3) @[[BAR]], ptr @kernel}
+; LDS metadata must have exactly one entry (the LDS struct), no barrier entries.
+; CHECK-DAG: !amdgpu.lds.uses = !{[[LDS_MD:![0-9]+]]}
+; CHECK-DAG: [[LDS_MD]] = !{ptr @kernel, ptr addrspace(3) @__amdgpu_lds.kernel}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-classify.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-classify.ll
new file mode 100644
index 0000000000000..bec74968599e9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-classify.ll
@@ -0,0 +1,73 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; Test the three-way classification of LDS variables:
+; 1. Global-scope (external linkage): standalone external declaration
+; 2. Kernel-scope (internal linkage, single kernel user): wrapped in per-kernel struct
+; 3. Callee-scope (internal linkage, single callee user): wrapped in per-callee struct
+;
+; Also tests that a global-scope variable used by multiple functions produces
+; one metadata entry per (function, variable) pair.
+
+; Global-scope: external linkage, used by both func and my_kernel.
+ at lds_global = addrspace(3) global [64 x i32] poison, align 16
+
+; Callee-scope: internal linkage, used only by func.
+ at lds_func_priv = internal addrspace(3) global [32 x float] poison, align 4
+
+; Kernel-scope: internal linkage, used only by my_kernel.
+ at lds_kernel_priv = internal addrspace(3) global [16 x i64] poison, align 8
+
+declare void @extern_func()
+
+; Global-scope: remains as external declaration.
+; CHECK-DAG: @lds_global = external addrspace(3) global [64 x i32]
+
+; Callee-scope: wrapped into per-function struct.
+; CHECK-DAG: @__amdgpu_lds.func = external {{(dso_local )?}}addrspace(3) global %__amdgpu_lds.func.t
+
+; Kernel-scope: wrapped into per-kernel struct.
+; CHECK-DAG: @__amdgpu_lds.my_kernel = external {{(dso_local )?}}addrspace(3) global %__amdgpu_lds.my_kernel.t
+
+; Original internal-linkage variables should be removed.
+; CHECK-NOT: @lds_func_priv
+; CHECK-NOT: @lds_kernel_priv
+
+; func: uses lds_global directly, uses lds_func_priv via struct GEP.
+; CHECK-LABEL: define void @func()
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @lds_global
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @__amdgpu_lds.func
+
+; my_kernel: uses lds_global directly, uses lds_kernel_priv via struct GEP.
+; CHECK-LABEL: define amdgpu_kernel void @my_kernel()
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @lds_global
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @__amdgpu_lds.my_kernel
+
+; Metadata:
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+, ![0-9]+, ![0-9]+, ![0-9]+}}}
+; Function-scope entries (one per struct).
+; CHECK-DAG: !{ptr @my_kernel, ptr addrspace(3) @__amdgpu_lds.my_kernel}
+; CHECK-DAG: !{ptr @func, ptr addrspace(3) @__amdgpu_lds.func}
+; Global-scope entries (one per using function).
+; CHECK-DAG: !{ptr @my_kernel, ptr addrspace(3) @lds_global}
+; CHECK-DAG: !{ptr @func, ptr addrspace(3) @lds_global}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define void @func() {
+ %gep1 = getelementptr [64 x i32], ptr addrspace(3) @lds_global, i32 0, i32 0
+ store i32 1, ptr addrspace(3) %gep1
+ %gep2 = getelementptr [32 x float], ptr addrspace(3) @lds_func_priv, i32 0, i32 0
+ store float 2.0, ptr addrspace(3) %gep2
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @my_kernel() {
+ %gep1 = getelementptr [64 x i32], ptr addrspace(3) @lds_global, i32 0, i32 0
+ store i32 3, ptr addrspace(3) %gep1
+ %gep2 = getelementptr [16 x i64], ptr addrspace(3) @lds_kernel_priv, i32 0, i32 0
+ store i64 4, ptr addrspace(3) %gep2
+ call void @func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-global-scope.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-global-scope.ll
new file mode 100644
index 0000000000000..b70fc9645e098
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-global-scope.ll
@@ -0,0 +1,125 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; Comprehensive test for global-scope (external linkage) LDS in link-time mode.
+; External LDS variables remain as standalone external declarations -- they are
+; NOT wrapped into per-function structs.
+;
+; Scenarios covered:
+; - Device function and kernel each using distinct global-scope LDS
+; - Multiple kernels sharing a device function that uses LDS
+; - A single function using multiple LDS variables
+; - Transitive call chain where only a leaf uses LDS
+; - Kernel directly using LDS (no device-function LDS user)
+
+; -- Global variables --
+ at lds_shared = addrspace(3) global [64 x i32] poison, align 16
+ at lds_kernel_a = addrspace(3) global [32 x float] poison, align 4
+ at lds_kernel_b = addrspace(3) global [16 x i64] poison, align 8
+ at lds_leaf = addrspace(3) global [8 x i32] poison, align 4
+ at lds_direct = addrspace(3) global [32 x float] poison, align 4
+
+declare void @extern_func()
+
+; All external-linkage LDS become external declarations.
+; CHECK-DAG: @lds_shared = external addrspace(3) global [64 x i32]
+; CHECK-DAG: @lds_kernel_a = external addrspace(3) global [32 x float]
+; CHECK-DAG: @lds_kernel_b = external addrspace(3) global [16 x i64]
+; CHECK-DAG: @lds_leaf = external addrspace(3) global [8 x i32]
+; CHECK-DAG: @lds_direct = external addrspace(3) global [32 x float]
+
+; No per-function structs should be created for any function.
+; CHECK-NOT: @__amdgpu_lds.shared_func
+; CHECK-NOT: @__amdgpu_lds.kernel_a
+; CHECK-NOT: @__amdgpu_lds.kernel_b
+; CHECK-NOT: @__amdgpu_lds.leaf_func
+; CHECK-NOT: @__amdgpu_lds.mid_func
+; CHECK-NOT: @__amdgpu_lds.direct_kernel
+
+; --- shared_func: uses lds_shared, called by both kernel_a and kernel_b ---
+; CHECK-LABEL: define void @shared_func()
+; CHECK: getelementptr [64 x i32], ptr addrspace(3) @lds_shared
+; CHECK: call void @extern_func()
+
+; --- kernel_a: uses its own LDS + calls shared_func ---
+; CHECK-LABEL: define amdgpu_kernel void @kernel_a()
+; CHECK: getelementptr [32 x float], ptr addrspace(3) @lds_kernel_a
+; CHECK: call void @shared_func()
+
+; --- kernel_b: uses its own LDS + calls shared_func ---
+; CHECK-LABEL: define amdgpu_kernel void @kernel_b()
+; CHECK: getelementptr [16 x i64], ptr addrspace(3) @lds_kernel_b
+; CHECK: call void @shared_func()
+
+; --- leaf_func: uses lds_leaf (transitive -- called via mid_func) ---
+; CHECK-LABEL: define void @leaf_func()
+; CHECK: getelementptr [8 x i32], ptr addrspace(3) @lds_leaf
+
+; --- mid_func: no LDS, just calls leaf_func + extern ---
+; CHECK-LABEL: define void @mid_func()
+; CHECK-NOT: @__amdgpu_lds
+; CHECK: call void @leaf_func()
+; CHECK: call void @extern_func()
+
+; --- transitive_kernel: calls mid_func (transitive LDS user) ---
+; CHECK-LABEL: define amdgpu_kernel void @transitive_kernel()
+; CHECK: call void @mid_func()
+
+; --- direct_kernel: kernel directly uses LDS, no device function uses LDS ---
+; CHECK-LABEL: define amdgpu_kernel void @direct_kernel()
+; CHECK: getelementptr [32 x float], ptr addrspace(3) @lds_direct
+
+; Metadata: one entry per (function, variable) pair for direct users only.
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+, ![0-9]+, ![0-9]+, ![0-9]+, ![0-9]+}}}
+; CHECK-DAG: !{ptr @shared_func, ptr addrspace(3) @lds_shared}
+; CHECK-DAG: !{ptr @kernel_a, ptr addrspace(3) @lds_kernel_a}
+; CHECK-DAG: !{ptr @kernel_b, ptr addrspace(3) @lds_kernel_b}
+; CHECK-DAG: !{ptr @leaf_func, ptr addrspace(3) @lds_leaf}
+; CHECK-DAG: !{ptr @direct_kernel, ptr addrspace(3) @lds_direct}
+
+; Module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define void @shared_func() {
+ %gep = getelementptr [64 x i32], ptr addrspace(3) @lds_shared, i32 0, i32 0
+ store i32 1, ptr addrspace(3) %gep
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @kernel_a() {
+ %gep = getelementptr [32 x float], ptr addrspace(3) @lds_kernel_a, i32 0, i32 0
+ store float 1.0, ptr addrspace(3) %gep
+ call void @shared_func()
+ ret void
+}
+
+define amdgpu_kernel void @kernel_b() {
+ %gep = getelementptr [16 x i64], ptr addrspace(3) @lds_kernel_b, i32 0, i32 0
+ store i64 1, ptr addrspace(3) %gep
+ call void @shared_func()
+ ret void
+}
+
+define void @leaf_func() {
+ %gep = getelementptr [8 x i32], ptr addrspace(3) @lds_leaf, i32 0, i32 0
+ store i32 42, ptr addrspace(3) %gep
+ ret void
+}
+
+define void @mid_func() {
+ call void @leaf_func()
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @transitive_kernel() {
+ call void @mid_func()
+ ret void
+}
+
+define amdgpu_kernel void @direct_kernel() {
+ %gep = getelementptr [32 x float], ptr addrspace(3) @lds_direct, i32 0, i32 0
+ store float 1.0, ptr addrspace(3) %gep
+ call void @extern_func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-func.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-func.ll
new file mode 100644
index 0000000000000..45391036cf50c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-func.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; An internal (static) device function uses an internal LDS variable.
+; The per-function struct must get a module-unique hash suffix to avoid
+; cross-TU name collisions (two TUs can both have "static void helper()").
+
+ at lds_priv = internal addrspace(3) global [32 x i32] poison, align 16
+
+declare void @extern_func()
+
+; Struct name includes the function name AND a module-unique hash.
+; CHECK: @[[STRUCT:__amdgpu_lds\.helper\.[a-f0-9]+]] = external {{(dso_local )?}}addrspace(3) global %[[STRUCT]].t, align 16
+
+; Original variable should be removed.
+; CHECK-NOT: @lds_priv
+
+; CHECK-LABEL: define internal void @helper()
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @[[STRUCT]]
+
+; CHECK-LABEL: define amdgpu_kernel void @kernel()
+; CHECK: call void @helper()
+
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+}}}
+; CHECK-DAG: !{ptr @helper, ptr addrspace(3) @[[STRUCT]]}
+
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define internal void @helper() {
+ %gep = getelementptr [32 x i32], ptr addrspace(3) @lds_priv, i32 0, i32 0
+ store i32 1, ptr addrspace(3) %gep
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @kernel() {
+ call void @helper()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-multi-user.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-multi-user.ll
new file mode 100644
index 0000000000000..212631676f290
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-multi-user.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+source_filename = "source_a.hip"
+
+; Internal-linkage LDS variables used by multiple kernels must be packed into a
+; per-module struct with a module-unique name, rather than promoted individually
+; to external linkage, to avoid cross-TU name collisions.
+
+ at a = internal addrspace(3) global [32 x i32] poison, align 16
+ at b = internal addrspace(3) global [16 x float] poison, align 4
+
+; Per-module struct containing both internal multi-user variables.
+; CHECK: @[[INTERN:__amdgpu_lds.__internal\.[a-f0-9]+]] = external {{(dso_local )?}}addrspace(3) global %[[INTERN]].t, align 16
+
+; Original internal-linkage variables should be removed.
+; CHECK-NOT: @a =
+; CHECK-NOT: @b =
+
+; Both kernels reference the struct.
+; CHECK-LABEL: define amdgpu_kernel void @kernel1()
+; CHECK: @[[INTERN]]
+; CHECK: @[[INTERN]]
+
+; CHECK-LABEL: define amdgpu_kernel void @kernel2()
+; CHECK: @[[INTERN]]
+; CHECK: @[[INTERN]]
+
+; Metadata: struct entries for both kernels.
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+, ![0-9]+}}}
+; CHECK-DAG: !{ptr @kernel1, ptr addrspace(3) @[[INTERN]]}
+; CHECK-DAG: !{ptr @kernel2, ptr addrspace(3) @[[INTERN]]}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define amdgpu_kernel void @kernel1() {
+ %gep_a = getelementptr [32 x i32], ptr addrspace(3) @a, i32 0, i32 0
+ store i32 1, ptr addrspace(3) %gep_a
+ %gep_b = getelementptr [16 x float], ptr addrspace(3) @b, i32 0, i32 0
+ store float 2.0, ptr addrspace(3) %gep_b
+ ret void
+}
+
+define amdgpu_kernel void @kernel2() {
+ %gep_a = getelementptr [32 x i32], ptr addrspace(3) @a, i32 0, i32 0
+ store i32 3, ptr addrspace(3) %gep_a
+ %gep_b = getelementptr [16 x float], ptr addrspace(3) @b, i32 0, i32 0
+ store float 4.0, ptr addrspace(3) %gep_b
+ ret void
+}
More information about the llvm-commits
mailing list