[llvm] [AMDGPU] Add object linking support for LDS and named barrier lowering in the middle end (PR #191645)
Shilei Tian via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 14:19:50 PDT 2026
https://github.com/shiltian updated https://github.com/llvm/llvm-project/pull/191645
>From 9962180da16d8c5a8431947b02c9aa4f0238d612 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i at tianshilei.me>
Date: Sat, 11 Apr 2026 12:29:13 -0400
Subject: [PATCH] [AMDGPU] Add object linking support for LDS and named barrier
lowering in the middle end
This is the first patch in a series introducing object linking support for
AMDGPU.
This PR adds the -amdgpu-enable-object-linking flag to enable object linking in
the backend. It also updates the AMDGPULowerModuleLDSPass and
AMDGPULowerExecSync passes to support lowering LDS and named barrier globals
when object linking is enabled.
---
.../lib/Target/AMDGPU/AMDGPULowerExecSync.cpp | 37 ++++
.../AMDGPU/AMDGPULowerModuleLDSPass.cpp | 163 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 7 +
llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h | 1 +
.../AMDGPU/lds-link-time-named-barrier.ll | 35 ++++
.../lower-module-lds-link-time-classify.ll | 73 ++++++++
...odule-lds-link-time-internal-multi-user.ll | 50 ++++++
...-module-lds-link-time-kernel-direct-lds.ll | 40 +++++
...lower-module-lds-link-time-multi-kernel.ll | 62 +++++++
...module-lds-link-time-multi-lds-per-func.ll | 52 ++++++
.../lower-module-lds-link-time-transitive.ll | 50 ++++++
.../AMDGPU/lower-module-lds-link-time.ll | 49 ++++++
12 files changed, 619 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-classify.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-multi-user.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-kernel-direct-lds.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-multi-kernel.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-multi-lds-per-func.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-transitive.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
index c26e97360efef..25850c9ed1417 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
@@ -178,7 +178,44 @@ static bool lowerExecSyncGlobalVariables(
return Changed;
}
+// With object linking, barrier ID assignment is deferred to the linker.
+// Externalize named barrier globals and emit self-contained metadata so the
+// AsmPrinter can generate the callgraph entries the linker needs.
+static bool handleNamedBarriersForObjectLinking(Module &M) {
+ DenseMap<GlobalVariable *, DenseSet<Function *>> BarrierToFuncs;
+ for (GlobalVariable &GV : M.globals()) {
+ if (!isNamedBarrier(GV) || GV.use_empty())
+ continue;
+ for (User *U : GV.users()) {
+ if (auto *I = dyn_cast<Instruction>(U))
+ BarrierToFuncs[&GV].insert(I->getFunction());
+ }
+ }
+ if (BarrierToFuncs.empty())
+ return false;
+
+ LLVMContext &Ctx = M.getContext();
+ NamedMDNode *BarMD = M.getOrInsertNamedMetadata("amdgpu.named_barrier.uses");
+
+ for (auto &[V, Funcs] : BarrierToFuncs) {
+ V->setInitializer(nullptr);
+ V->setLinkage(GlobalValue::ExternalLinkage);
+ if (!V->getName().starts_with("__amdgpu_named_barrier"))
+ V->setName("__amdgpu_named_barrier." + V->getName());
+
+ SmallVector<Metadata *, 4> Ops;
+ Ops.push_back(ValueAsMetadata::get(V));
+ for (Function *F : Funcs)
+ Ops.push_back(ValueAsMetadata::get(F));
+ BarMD->addOperand(MDNode::get(Ctx, Ops));
+ }
+ return true;
+}
+
static bool runLowerExecSyncGlobals(Module &M) {
+ if (AMDGPUTargetMachine::EnableObjectLinking)
+ return handleNamedBarriersForObjectLinking(M);
+
CallGraph CG = CallGraph(M);
bool Changed = false;
Changed |= eliminateConstantExprUsesOfLDSFromAllInstructions(M);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index 699cf7d4d1cdd..87095604f33d8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -185,6 +185,7 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SmallString.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -910,7 +911,169 @@ class AMDGPULowerModuleLDS {
return KernelToCreatedDynamicLDS;
}
+ // Per-TU mode for link-time LDS resolution. Instead of computing a global
+ // layout, create per-function LDS struct declarations so the linker can
+ // assign offsets across TUs.
+ bool runOnModuleLinkTime(Module &M) {
+ bool Changed = superAlignLDSGlobals(M);
+ Changed |= eliminateConstantExprUsesOfLDSFromAllInstructions(M);
+
+ CallGraph CG(M);
+ FunctionVariableMap KernelLDSUses, FunctionLDSUses;
+ getUsesOfLDSByFunction(CG, M, KernelLDSUses, FunctionLDSUses);
+
+ if (KernelLDSUses.empty() && FunctionLDSUses.empty())
+ return Changed;
+
+ FunctionVariableMap AllLDSUses;
+ for (auto &[F, Vars] : KernelLDSUses)
+ AllLDSUses[F].insert(Vars.begin(), Vars.end());
+ for (auto &[F, Vars] : FunctionLDSUses)
+ AllLDSUses[F].insert(Vars.begin(), Vars.end());
+
+ // Named barriers are handled by AMDGPULowerExecSync; filter them out.
+ for (auto &[F, Vars] : AllLDSUses) {
+ SmallVector<GlobalVariable *> Barriers;
+ for (GlobalVariable *V : Vars)
+ if (AMDGPU::isNamedBarrier(*V))
+ Barriers.push_back(V);
+ for (GlobalVariable *V : Barriers)
+ Vars.erase(V);
+ }
+
+ // Build reverse map: LDS variable -> functions that use it.
+ DenseMap<GlobalVariable *, SmallVector<Function *, 4>> VarToFuncs;
+ for (auto &[F, Vars] : AllLDSUses) {
+ for (GlobalVariable *V : Vars)
+ VarToFuncs[V].push_back(F);
+ }
+
+ // A variable is function-scope iff it has local linkage and exactly one
+ // user function. Everything else is global-scope and must remain as a
+ // standalone external declaration so the linker can assign a single shared
+ // offset.
+ DenseSet<GlobalVariable *> GlobalScopeVars;
+ DenseSet<GlobalVariable *> InternalMultiUserVars;
+ for (auto &[V, Funcs] : VarToFuncs) {
+ if (!V->hasLocalLinkage() || Funcs.size() > 1) {
+ GlobalScopeVars.insert(V);
+ if (V->hasLocalLinkage())
+ InternalMultiUserVars.insert(V);
+ }
+ }
+
+ // Wrap function-scope LDS into per-function structs (unchanged logic,
+ // but global-scope variables are excluded from the set).
+ SmallVector<std::pair<Function *, GlobalVariable *>, 4> FuncToLdsStruct;
+ DenseSet<GlobalVariable *> AllReplacedVars;
+ for (auto &KV : AllLDSUses) {
+ Function *F = KV.first;
+ DenseSet<GlobalVariable *> FuncScopeVars;
+ for (GlobalVariable *V : KV.second)
+ if (!GlobalScopeVars.count(V))
+ FuncScopeVars.insert(V);
+
+ if (FuncScopeVars.empty())
+ continue;
+
+ std::string StructName = ("__amdgpu_lds." + F->getName()).str();
+ LDSVariableReplacement Replacement =
+ createLDSVariableReplacement(M, StructName, FuncScopeVars);
+
+ GlobalVariable *SGV = Replacement.SGV;
+ SGV->setLinkage(GlobalValue::ExternalLinkage);
+ SGV->setInitializer(nullptr);
+ FuncToLdsStruct.push_back({F, SGV});
+
+ replaceLDSVariablesWithStruct(
+ M, FuncScopeVars, Replacement, [F](const Use &U) {
+ auto *I = dyn_cast<Instruction>(U.getUser());
+ return I && I->getFunction() == F;
+ });
+
+ AllReplacedVars.insert(FuncScopeVars.begin(), FuncScopeVars.end());
+ }
+
+ // Internal-linkage LDS variables used by multiple functions would collide
+ // across TUs if promoted individually to external linkage (same name in
+ // different TUs). Pack them into a single per-module struct with a
+ // module-unique name so the linker treats them as one allocation unit.
+ if (!InternalMultiUserVars.empty()) {
+ std::string ModuleId = getUniqueModuleId(&M);
+ assert(!ModuleId.empty() &&
+ "modules with LDS variables should have a unique ID");
+ std::string StructName = "__amdgpu_lds.__internal" + ModuleId;
+
+ LDSVariableReplacement Replacement =
+ createLDSVariableReplacement(M, StructName, InternalMultiUserVars);
+
+ GlobalVariable *SGV = Replacement.SGV;
+ SGV->setLinkage(GlobalValue::ExternalLinkage);
+ SGV->setInitializer(nullptr);
+
+ replaceLDSVariablesWithStruct(
+ M, InternalMultiUserVars, Replacement,
+ [](const Use &U) { return isa<Instruction>(U.getUser()); });
+
+ DenseSet<Function *> FuncsUsingInternalVars;
+ for (GlobalVariable *V : InternalMultiUserVars) {
+ for (Function *F : VarToFuncs[V])
+ FuncsUsingInternalVars.insert(F);
+ }
+ for (Function *F : FuncsUsingInternalVars)
+ FuncToLdsStruct.push_back({F, SGV});
+
+ AllReplacedVars.insert(InternalMultiUserVars.begin(),
+ InternalMultiUserVars.end());
+ }
+
+ // Convert global-scope LDS to external declarations. Their uses remain
+ // intact and ISel generates R_AMDGPU_ABS32_LO relocations for them.
+ for (GlobalVariable *V : GlobalScopeVars) {
+ V->setInitializer(nullptr);
+ V->setLinkage(GlobalValue::ExternalLinkage);
+ }
+
+ // Emit amdgpu.lds.uses metadata for struct and global-scope LDS.
+ {
+ LLVMContext &Ctx = M.getContext();
+ NamedMDNode *LdsMD = M.getOrInsertNamedMetadata("amdgpu.lds.uses");
+
+ for (auto &[F, SGV] : FuncToLdsStruct)
+ LdsMD->addOperand(MDNode::get(
+ Ctx, {ValueAsMetadata::get(F), ValueAsMetadata::get(SGV)}));
+
+ for (auto &[V, Funcs] : VarToFuncs) {
+ if (GlobalScopeVars.count(V) && !InternalMultiUserVars.count(V)) {
+ for (Function *F : Funcs) {
+ LdsMD->addOperand(MDNode::get(
+ Ctx, {ValueAsMetadata::get(F), ValueAsMetadata::get(V)}));
+ }
+ }
+ }
+ }
+
+ M.addModuleFlag(Module::Error, "amdgpu-link-time-lds", 1);
+
+ DenseSet<GlobalVariable *> AllLDSVarsForCleanup = AllReplacedVars;
+ AllLDSVarsForCleanup.insert(GlobalScopeVars.begin(), GlobalScopeVars.end());
+ removeLocalVarsFromUsedLists(M, AllLDSVarsForCleanup);
+ for (GlobalVariable *GV : AllReplacedVars) {
+ GV->removeDeadConstantUsers();
+ if (GV->use_empty())
+ GV->eraseFromParent();
+ }
+
+ return true;
+ }
+
bool runOnModule(Module &M) {
+ if (AMDGPUTargetMachine::EnableObjectLinking)
+ return runOnModuleLinkTime(M);
+ return runOnModuleNormal(M);
+ }
+
+ bool runOnModuleNormal(Module &M) {
CallGraph CG = CallGraph(M);
bool Changed = superAlignLDSGlobals(M);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index f0d3e7056f69e..e08345f5acbc5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -546,6 +546,12 @@ static cl::opt<bool>
"and asan instrument resulting IR."),
cl::init(true), cl::Hidden);
+static cl::opt<bool, true> EnableObjectLinking(
+ "amdgpu-enable-object-linking",
+ cl::desc("Enable object linking for cross-TU LDS and ABI support"),
+ cl::location(AMDGPUTargetMachine::EnableObjectLinking), cl::init(false),
+ cl::Hidden);
+
static cl::opt<bool, true> EnableLowerModuleLDS(
"amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),
cl::location(AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(true),
@@ -877,6 +883,7 @@ AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
}
bool AMDGPUTargetMachine::EnableFunctionCalls = false;
+bool AMDGPUTargetMachine::EnableObjectLinking = false;
bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;
AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 1be3e5f98ebb8..e2c27f3822380 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -40,6 +40,7 @@ class AMDGPUTargetMachine : public CodeGenTargetMachineImpl {
public:
static bool EnableFunctionCalls;
+ static bool EnableObjectLinking;
static bool EnableLowerModuleLDS;
AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
new file mode 100644
index 0000000000000..b6c86089c4832
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-enable-object-linking -passes=amdgpu-lower-exec-sync,amdgpu-lower-module-lds < %s | FileCheck %s
+
+; Verify that with object linking enabled:
+; 1. AMDGPULowerExecSync externalizes named barriers and emits
+; amdgpu.named_barrier.uses metadata with (barrier, func...) format
+; 2. AMDGPULowerModuleLDS does not handle named barriers at all
+; 3. amdgpu.lds.uses does NOT contain barrier entries
+
+ at bar = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
+ at lds = internal addrspace(3) global [4 x i32] poison, align 4
+
+; Named barrier becomes an external declaration with __amdgpu_named_barrier prefix.
+; CHECK: @__amdgpu_named_barrier.bar = external dso_local addrspace(3) global target("amdgcn.named.barrier", 0)
+; CHECK-NOT: !absolute_symbol
+; Regular LDS is packed into the per-function struct (external, for linker).
+; CHECK: @__amdgpu_lds.kernel = external dso_local addrspace(3) global %__amdgpu_lds.kernel.t, align 16
+
+define amdgpu_kernel void @kernel(i32 %idx) {
+; CHECK-LABEL: define amdgpu_kernel void @kernel(
+; CHECK: call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @__amdgpu_named_barrier.bar, i32 3)
+; CHECK: call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @__amdgpu_named_barrier.bar)
+ call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar, i32 3)
+ call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar)
+ call void @llvm.amdgcn.s.barrier.wait(i16 1)
+ %gep = getelementptr [4 x i32], ptr addrspace(3) @lds, i32 0, i32 %idx
+ store i32 42, ptr addrspace(3) %gep, align 4
+ ret void
+}
+
+; Named barrier metadata: (barrier_sym, func1, ...) -- emitted by ExecSync.
+; CHECK-DAG: !amdgpu.named_barrier.uses = !{[[BAR_MD:![0-9]+]]}
+; CHECK-DAG: [[BAR_MD]] = !{ptr addrspace(3) @__amdgpu_named_barrier.bar, ptr @kernel}
+; LDS metadata must have exactly one entry (the LDS struct), no barrier entries.
+; CHECK-DAG: !amdgpu.lds.uses = !{[[LDS_MD:![0-9]+]]}
+; CHECK-DAG: [[LDS_MD]] = !{ptr @kernel, ptr addrspace(3) @__amdgpu_lds.kernel}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-classify.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-classify.ll
new file mode 100644
index 0000000000000..bec74968599e9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-classify.ll
@@ -0,0 +1,73 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; Test the three-way classification of LDS variables:
+; 1. Global-scope (external linkage): standalone external declaration
+; 2. Kernel-scope (internal linkage, single kernel user): wrapped in per-kernel struct
+; 3. Callee-scope (internal linkage, single callee user): wrapped in per-callee struct
+;
+; Also tests that a global-scope variable used by multiple functions produces
+; one metadata entry per (function, variable) pair.
+
+; Global-scope: external linkage, used by both func and my_kernel.
+ at lds_global = addrspace(3) global [64 x i32] poison, align 16
+
+; Callee-scope: internal linkage, used only by func.
+ at lds_func_priv = internal addrspace(3) global [32 x float] poison, align 4
+
+; Kernel-scope: internal linkage, used only by my_kernel.
+ at lds_kernel_priv = internal addrspace(3) global [16 x i64] poison, align 8
+
+declare void @extern_func()
+
+; Global-scope: remains as external declaration.
+; CHECK-DAG: @lds_global = external addrspace(3) global [64 x i32]
+
+; Callee-scope: wrapped into per-function struct.
+; CHECK-DAG: @__amdgpu_lds.func = external {{(dso_local )?}}addrspace(3) global %__amdgpu_lds.func.t
+
+; Kernel-scope: wrapped into per-kernel struct.
+; CHECK-DAG: @__amdgpu_lds.my_kernel = external {{(dso_local )?}}addrspace(3) global %__amdgpu_lds.my_kernel.t
+
+; Original internal-linkage variables should be removed.
+; CHECK-NOT: @lds_func_priv
+; CHECK-NOT: @lds_kernel_priv
+
+; func: uses lds_global directly, uses lds_func_priv via struct GEP.
+; CHECK-LABEL: define void @func()
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @lds_global
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @__amdgpu_lds.func
+
+; my_kernel: uses lds_global directly, uses lds_kernel_priv via struct GEP.
+; CHECK-LABEL: define amdgpu_kernel void @my_kernel()
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @lds_global
+; CHECK: getelementptr {{.*}} ptr addrspace(3) @__amdgpu_lds.my_kernel
+
+; Metadata:
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+, ![0-9]+, ![0-9]+, ![0-9]+}}}
+; Function-scope entries (one per struct).
+; CHECK-DAG: !{ptr @my_kernel, ptr addrspace(3) @__amdgpu_lds.my_kernel}
+; CHECK-DAG: !{ptr @func, ptr addrspace(3) @__amdgpu_lds.func}
+; Global-scope entries (one per using function).
+; CHECK-DAG: !{ptr @my_kernel, ptr addrspace(3) @lds_global}
+; CHECK-DAG: !{ptr @func, ptr addrspace(3) @lds_global}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define void @func() {
+ %gep1 = getelementptr [64 x i32], ptr addrspace(3) @lds_global, i32 0, i32 0
+ store i32 1, ptr addrspace(3) %gep1
+ %gep2 = getelementptr [32 x float], ptr addrspace(3) @lds_func_priv, i32 0, i32 0
+ store float 2.0, ptr addrspace(3) %gep2
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @my_kernel() {
+ %gep1 = getelementptr [64 x i32], ptr addrspace(3) @lds_global, i32 0, i32 0
+ store i32 3, ptr addrspace(3) %gep1
+ %gep2 = getelementptr [16 x i64], ptr addrspace(3) @lds_kernel_priv, i32 0, i32 0
+ store i64 4, ptr addrspace(3) %gep2
+ call void @func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-multi-user.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-multi-user.ll
new file mode 100644
index 0000000000000..212631676f290
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-internal-multi-user.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+source_filename = "source_a.hip"
+
+; Internal-linkage LDS variables used by multiple kernels must be packed into a
+; per-module struct with a module-unique name, rather than promoted individually
+; to external linkage, to avoid cross-TU name collisions.
+
+ at a = internal addrspace(3) global [32 x i32] poison, align 16
+ at b = internal addrspace(3) global [16 x float] poison, align 4
+
+; Per-module struct containing both internal multi-user variables.
+; CHECK: @[[INTERN:__amdgpu_lds.__internal\.[a-f0-9]+]] = external {{(dso_local )?}}addrspace(3) global %[[INTERN]].t, align 16
+
+; Original internal-linkage variables should be removed.
+; CHECK-NOT: @a =
+; CHECK-NOT: @b =
+
+; Both kernels reference the struct.
+; CHECK-LABEL: define amdgpu_kernel void @kernel1()
+; CHECK: @[[INTERN]]
+; CHECK: @[[INTERN]]
+
+; CHECK-LABEL: define amdgpu_kernel void @kernel2()
+; CHECK: @[[INTERN]]
+; CHECK: @[[INTERN]]
+
+; Metadata: struct entries for both kernels.
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+, ![0-9]+}}}
+; CHECK-DAG: !{ptr @kernel1, ptr addrspace(3) @[[INTERN]]}
+; CHECK-DAG: !{ptr @kernel2, ptr addrspace(3) @[[INTERN]]}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define amdgpu_kernel void @kernel1() {
+ %gep_a = getelementptr [32 x i32], ptr addrspace(3) @a, i32 0, i32 0
+ store i32 1, ptr addrspace(3) %gep_a
+ %gep_b = getelementptr [16 x float], ptr addrspace(3) @b, i32 0, i32 0
+ store float 2.0, ptr addrspace(3) %gep_b
+ ret void
+}
+
+define amdgpu_kernel void @kernel2() {
+ %gep_a = getelementptr [32 x i32], ptr addrspace(3) @a, i32 0, i32 0
+ store i32 3, ptr addrspace(3) %gep_a
+ %gep_b = getelementptr [16 x float], ptr addrspace(3) @b, i32 0, i32 0
+ store float 4.0, ptr addrspace(3) %gep_b
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-kernel-direct-lds.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-kernel-direct-lds.ll
new file mode 100644
index 0000000000000..776e953c0fc3b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-kernel-direct-lds.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; Only the kernel itself directly uses LDS (no device function uses LDS).
+; The LDS variable has external linkage -> global-scope -> standalone declaration.
+
+ at lds_var = addrspace(3) global [32 x float] poison, align 4
+
+declare void @extern_func()
+
+; Global-scope: remains as external declaration.
+; CHECK: @lds_var = external addrspace(3) global [32 x float]
+; No per-function struct (variable is global-scope).
+; CHECK-NOT: @__amdgpu_lds.my_kernel
+; CHECK-NOT: @__amdgpu_lds.device_func
+
+; CHECK-LABEL: define void @device_func()
+; CHECK: call void @extern_func()
+
+; CHECK-LABEL: define amdgpu_kernel void @my_kernel()
+; CHECK: getelementptr [32 x float], ptr addrspace(3) @lds_var
+; CHECK: call void @device_func()
+
+; Per-function LDS ownership metadata.
+; CHECK: !amdgpu.lds.uses = !{[[LDS_MD:![0-9]+]]}
+; CHECK: [[LDS_MD]] = !{ptr @my_kernel, ptr addrspace(3) @lds_var}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define void @device_func() {
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @my_kernel() {
+ %gep = getelementptr [32 x float], ptr addrspace(3) @lds_var, i32 0, i32 0
+ store float 1.0, ptr addrspace(3) %gep
+ call void @device_func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-multi-kernel.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-multi-kernel.ll
new file mode 100644
index 0000000000000..3ae8a10f5004e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-multi-kernel.ll
@@ -0,0 +1,62 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; Two kernels share a device function that uses LDS and calls an external.
+; Each kernel also has its own external-linkage LDS variable.
+; All variables are global-scope (external linkage) -> standalone declarations.
+
+ at lds_shared = addrspace(3) global [64 x i32] poison, align 16
+ at lds_kernel_a = addrspace(3) global [32 x float] poison, align 4
+ at lds_kernel_b = addrspace(3) global [16 x i64] poison, align 8
+
+declare void @extern_func()
+
+; All become external declarations.
+; CHECK-DAG: @lds_shared = external addrspace(3) global [64 x i32]
+; CHECK-DAG: @lds_kernel_a = external addrspace(3) global [32 x float]
+; CHECK-DAG: @lds_kernel_b = external addrspace(3) global [16 x i64]
+; No per-function structs.
+; CHECK-NOT: @__amdgpu_lds.shared_func
+; CHECK-NOT: @__amdgpu_lds.kernel_a
+; CHECK-NOT: @__amdgpu_lds.kernel_b
+
+; CHECK-LABEL: define void @shared_func()
+; CHECK: getelementptr [64 x i32], ptr addrspace(3) @lds_shared
+; CHECK: call void @extern_func()
+
+; CHECK-LABEL: define amdgpu_kernel void @kernel_a()
+; CHECK: getelementptr [32 x float], ptr addrspace(3) @lds_kernel_a
+; CHECK: call void @shared_func()
+
+; CHECK-LABEL: define amdgpu_kernel void @kernel_b()
+; CHECK: getelementptr [16 x i64], ptr addrspace(3) @lds_kernel_b
+; CHECK: call void @shared_func()
+
+; Per-function LDS ownership metadata for global-scope vars.
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+, ![0-9]+, ![0-9]+}}}
+; CHECK-DAG: !{ptr @shared_func, ptr addrspace(3) @lds_shared}
+; CHECK-DAG: !{ptr @kernel_a, ptr addrspace(3) @lds_kernel_a}
+; CHECK-DAG: !{ptr @kernel_b, ptr addrspace(3) @lds_kernel_b}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define void @shared_func() {
+ %gep = getelementptr [64 x i32], ptr addrspace(3) @lds_shared, i32 0, i32 0
+ store i32 1, ptr addrspace(3) %gep
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @kernel_a() {
+ %gep = getelementptr [32 x float], ptr addrspace(3) @lds_kernel_a, i32 0, i32 0
+ store float 1.0, ptr addrspace(3) %gep
+ call void @shared_func()
+ ret void
+}
+
+define amdgpu_kernel void @kernel_b() {
+ %gep = getelementptr [16 x i64], ptr addrspace(3) @lds_kernel_b, i32 0, i32 0
+ store i64 1, ptr addrspace(3) %gep
+ call void @shared_func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-multi-lds-per-func.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-multi-lds-per-func.ll
new file mode 100644
index 0000000000000..f6f0e71ec62ae
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-multi-lds-per-func.ll
@@ -0,0 +1,52 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; A single device function uses three external-linkage LDS variables with
+; different sizes and alignments. All are global-scope -> standalone
+; declarations (not wrapped into a struct).
+
+ at lds_small = addrspace(3) global [4 x i8] poison, align 1
+ at lds_medium = addrspace(3) global [64 x i32] poison, align 16
+ at lds_large = addrspace(3) global [128 x i64] poison, align 8
+
+declare void @extern_func()
+
+; All become external declarations.
+; CHECK-DAG: @lds_small = external addrspace(3) global [4 x i8]
+; CHECK-DAG: @lds_medium = external addrspace(3) global [64 x i32]
+; CHECK-DAG: @lds_large = external addrspace(3) global [128 x i64]
+; No per-function struct.
+; CHECK-NOT: @__amdgpu_lds.device_func
+
+; Uses remain as direct references.
+; CHECK-LABEL: define void @device_func()
+; CHECK: getelementptr [4 x i8], ptr addrspace(3) @lds_small
+; CHECK: store
+; CHECK: getelementptr [64 x i32], ptr addrspace(3) @lds_medium
+; CHECK: store
+; CHECK: getelementptr [128 x i64], ptr addrspace(3) @lds_large
+; CHECK: store
+
+; Per-function metadata for each (function, variable) pair.
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+, ![0-9]+, ![0-9]+}}}
+; CHECK-DAG: !{ptr @device_func, ptr addrspace(3) @lds_small}
+; CHECK-DAG: !{ptr @device_func, ptr addrspace(3) @lds_medium}
+; CHECK-DAG: !{ptr @device_func, ptr addrspace(3) @lds_large}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define void @device_func() {
+ %gep1 = getelementptr [4 x i8], ptr addrspace(3) @lds_small, i32 0, i32 0
+ store i8 1, ptr addrspace(3) %gep1
+ %gep2 = getelementptr [64 x i32], ptr addrspace(3) @lds_medium, i32 0, i32 0
+ store i32 2, ptr addrspace(3) %gep2
+ %gep3 = getelementptr [128 x i64], ptr addrspace(3) @lds_large, i32 0, i32 0
+ store i64 3, ptr addrspace(3) %gep3
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @my_kernel() {
+ call void @device_func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-transitive.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-transitive.ll
new file mode 100644
index 0000000000000..86bbe6614bd2a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time-transitive.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; Transitive call chain: kernel -> mid_func -> leaf_func.
+; Only leaf_func uses LDS. mid_func calls an external. The LDS variable
+; has external linkage -> global-scope -> standalone declaration.
+
+ at lds_var = addrspace(3) global [64 x i32] poison, align 16
+
+declare void @extern_func()
+
+; Global-scope: remains as external declaration.
+; CHECK: @lds_var = external addrspace(3) global [64 x i32]
+; No per-function struct.
+; CHECK-NOT: @__amdgpu_lds.leaf_func
+
+; CHECK-LABEL: define void @leaf_func()
+; CHECK: getelementptr [64 x i32], ptr addrspace(3) @lds_var
+
+; mid_func does not use LDS directly — should have no LDS struct.
+; CHECK-LABEL: define void @mid_func()
+; CHECK-NOT: @__amdgpu_lds.mid_func
+; CHECK: call void @leaf_func()
+; CHECK: call void @extern_func()
+
+; CHECK-LABEL: define amdgpu_kernel void @top_kernel()
+; CHECK: call void @mid_func()
+
+; Metadata: leaf_func uses lds_var.
+; CHECK: !amdgpu.lds.uses = !{[[LDS_MD:![0-9]+]]}
+; CHECK: [[LDS_MD]] = !{ptr @leaf_func, ptr addrspace(3) @lds_var}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define void @leaf_func() {
+ %gep = getelementptr [64 x i32], ptr addrspace(3) @lds_var, i32 0, i32 0
+ store i32 42, ptr addrspace(3) %gep
+ ret void
+}
+
+define void @mid_func() {
+ call void @leaf_func()
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @top_kernel() {
+ call void @mid_func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time.ll
new file mode 100644
index 0000000000000..88825e3c7ff6e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-link-time.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-module-lds -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; Test that link-time LDS mode converts external-linkage LDS variables to
+; standalone external declarations (global-scope LDS) instead of wrapping
+; them into per-function structs.
+
+ at lds_a = addrspace(3) global [64 x i32] poison, align 16
+ at lds_b = addrspace(3) global [32 x float] poison, align 4
+
+declare void @extern_func()
+
+; Global-scope LDS variables become external declarations.
+; CHECK-DAG: @lds_a = external addrspace(3) global [64 x i32]
+; CHECK-DAG: @lds_b = external addrspace(3) global [32 x float]
+; No per-function LDS structs (all variables are global-scope).
+; CHECK-NOT: @__amdgpu_lds.device_func
+; CHECK-NOT: @__amdgpu_lds.kernel_func
+
+; Uses remain as direct references to the original variables.
+; CHECK-LABEL: define void @device_func()
+; CHECK: getelementptr [64 x i32], ptr addrspace(3) @lds_a
+; CHECK: store i32 1, ptr addrspace(3) %gep
+
+; CHECK-LABEL: define amdgpu_kernel void @kernel_func()
+; CHECK: getelementptr [32 x float], ptr addrspace(3) @lds_b
+; CHECK: store float 2.0{{.*}}, ptr addrspace(3) %gep
+
+; Per-function LDS ownership metadata for global-scope variables.
+; CHECK: !amdgpu.lds.uses = !{{{![0-9]+, ![0-9]+}}}
+; CHECK-DAG: !{ptr @device_func, ptr addrspace(3) @lds_a}
+; CHECK-DAG: !{ptr @kernel_func, ptr addrspace(3) @lds_b}
+
+; Module should be marked with the link-time LDS module flag.
+; CHECK: !{i32 1, !"amdgpu-link-time-lds", i32 1}
+
+define void @device_func() {
+ %gep = getelementptr [64 x i32], ptr addrspace(3) @lds_a, i32 0, i32 0
+ store i32 1, ptr addrspace(3) %gep
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @kernel_func() {
+ %gep = getelementptr [32 x float], ptr addrspace(3) @lds_b, i32 0, i32 0
+ store float 2.0, ptr addrspace(3) %gep
+ call void @device_func()
+ ret void
+}
More information about the llvm-commits
mailing list