[llvm] [llvm][Inliner] Don't inline functions referencing local COMDAT globals (PR #202687)

[via llvm-commits]

https://github.com/8051Enthusiast created https://github.com/llvm/llvm-project/pull/202687

It is not in general possible to inline functions in a comdat that reference other symbols of the group with local linkage: If we inline them into a function outside of the group, the caller will directly contain references to a symbol with comdat and local linkage. When the comdat subsequently gets dropped by the linker, the caller will remain with the reference to the symbol, and it will not get a replacement with an equivalent symbol of the actually chosen comdat, because the reference is local to the module.

We search inline candidate functions for such local references and rejects candidates containing those references. In a stage 2 linux build, no such rejections happen as clang does not seem to produce local symbols with comdat.

>From 074a43ed22d25bb16ef4c81e34efcddb3938d39e Mon Sep 17 00:00:00 2001
From: 8051enthusiast <8051enthusiast at protonmail.com>
Date: Tue, 9 Jun 2026 16:43:26 +0200
Subject: [PATCH] [llvm][Inliner] Don't inline functions referencing local
 COMDAT globals

It is not in general possible to inline functions in a comdat that
reference other symbols of the group with local linkage:
If we inline them into a function outside of the group, the caller will
directly contain references to a symbol with comdat and local linkage.
When the comdat subsequently gets dropped by the linker, the caller
will remain with the reference to the symbol, and it will not get a
replacement with an equivalent symbol of the actually chosen comdat,
because the reference is local to the module.

We search inline candidate functions for such local references
and rejects candidates containing those references.
In a stage 2 linux build, no such rejections happen as clang does not
seem to produce local symbols with comdat.
---
 llvm/lib/Transforms/Utils/InlineFunction.cpp | 61 +++++++++++++
 llvm/test/Transforms/Inline/comdat-local.ll  | 92 ++++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 llvm/test/Transforms/Inline/comdat-local.ll

diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index be186ffbf7e42..b7107066d8b7e 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -182,6 +182,58 @@ namespace {
   };
 } // end anonymous namespace
 
+static bool hasLocalComdatReferences(Value *V, Comdat *ParentComdat) {
+  auto *C = dyn_cast<Constant>(V);
+  if (!C || isa<ConstantData>(C))
+    return false;
+
+  if (auto *GV = dyn_cast<GlobalValue>(C))
+    return GV->hasLocalLinkage() && GV->getComdat() == ParentComdat;
+
+  if (auto *BBAddr = dyn_cast<BlockAddress>(C))
+    return BBAddr->getFunction()->hasLocalLinkage() &&
+           BBAddr->getFunction()->getComdat() == ParentComdat;
+
+  for (auto *Operand : C->operand_values()) {
+    if (hasLocalComdatReferences(Operand, ParentComdat))
+      return true;
+  }
+
+  return false;
+}
+
+static bool hasLocalComdatReferences(Function *F) {
+  auto *Comdat = F->getComdat();
+  // A non-comdat function cannot legally use globals
+  // with comdat and local linkage.
+  if (!Comdat)
+    return false;
+
+  auto HasLocalComdat = false;
+  for (auto *GO : Comdat->getUsers()) {
+    if (GO->hasLocalLinkage()) {
+      HasLocalComdat = true;
+      break;
+    }
+  }
+
+  // We cannot legally have references to comdat globals with local linkage if
+  // there are none in the current comdat.
+  if (!HasLocalComdat)
+    return false;
+
+  for (auto &BB : *F) {
+    for (auto &I : BB) {
+      for (auto *V : I.operand_values()) {
+        if (hasLocalComdatReferences(V, Comdat))
+          return true;
+      }
+    }
+  }
+
+  return false;
+}
+
 static IntrinsicInst *getConvergenceEntry(BasicBlock &BB) {
   BasicBlock::iterator It = BB.getFirstNonPHIIt();
   while (It != BB.end()) {
@@ -2710,6 +2762,15 @@ llvm::InlineResult llvm::CanInlineCallSite(const CallBase &CB,
     }
   }
 
+  // If we inline a function from another comdat group and the callee has
+  // references to global values that have both local linkage and comdat, we
+  // would retain references to those values from outside the comdat group. If
+  // the comdat group gets subsequently dropped, we would get a linker error.
+  if (Caller->getComdat() != CalledFunc->getComdat() &&
+      hasLocalComdatReferences(CalledFunc))
+    return InlineResult::failure(
+        "reference to object with local linkage and COMDAT inside function");
+
   return InlineResult::success();
 }
 
diff --git a/llvm/test/Transforms/Inline/comdat-local.ll b/llvm/test/Transforms/Inline/comdat-local.ll
new file mode 100644
index 0000000000000..92981a38f3a54
--- /dev/null
+++ b/llvm/test/Transforms/Inline/comdat-local.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=inline -S < %s | FileCheck %s
+$callee_global = comdat any
+
+$callee_global_gep = comdat any
+
+$comdat_caller = comdat any
+
+$callee_subcall = comdat any
+
+$callee_unused_private = comdat any
+
+ at global1 = private constant i8 1, comdat($callee_global)
+ at global2 = private constant i8 2, comdat($callee_global_gep)
+ at global3 = private constant i8 3, comdat($comdat_caller)
+ at global4 = private constant i8 4, comdat($callee_unused_private)
+
+define linkonce_odr ptr @callee_global() comdat {
+; CHECK-LABEL: define linkonce_odr ptr @callee_global() comdat {
+; CHECK-NEXT:    ret ptr @global1
+;
+  ret ptr @global1
+}
+
+define linkonce_odr ptr @callee_global_gep() comdat {
+; CHECK-LABEL: define linkonce_odr ptr @callee_global_gep() comdat {
+; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @global2, i64 1)
+;
+  ret ptr getelementptr inbounds (i8, ptr @global2, i64 1)
+}
+
+define private i32 @subcall() noinline comdat($callee_subcall) {
+; CHECK-LABEL: define private i32 @subcall(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] comdat($callee_subcall) {
+; CHECK-NEXT:    ret i32 10
+;
+  ret i32 10
+}
+
+define linkonce_odr i32 @callee_subcall() comdat {
+; CHECK-LABEL: define linkonce_odr i32 @callee_subcall() comdat {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @subcall()
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @subcall()
+  ret i32 %res
+}
+
+; If @callee_global was inlined into @caller, @caller would contain a reference to
+; @global1, which can disappear if the comdat group $callee_global gets dropped
+define i32 @caller() {
+; CHECK-LABEL: define i32 @caller() {
+; CHECK-NEXT:    [[RES1:%.*]] = call ptr @callee_global()
+; CHECK-NEXT:    [[RES2:%.*]] = call ptr @callee_global_gep()
+; CHECK-NEXT:    [[RES3:%.*]] = call i32 @callee_subcall()
+; CHECK-NEXT:    ret i32 [[RES3]]
+;
+  %res1 = call ptr @callee_global()
+  %res2 = call ptr @callee_global_gep()
+  %res3 = call i32 @callee_subcall()
+  ret i32 %res3
+}
+
+define linkonce_odr i32 @callee_unused_private() comdat {
+; CHECK-LABEL: define linkonce_odr i32 @callee_unused_private() comdat {
+; CHECK-NEXT:    ret i32 20
+;
+  ret i32 20
+}
+
+define i32 @caller_inline_success() {
+; CHECK-LABEL: define i32 @caller_inline_success() {
+; CHECK-NEXT:    ret i32 20
+;
+  %res = call i32 @callee_unused_private()
+  ret i32 %res
+}
+
+define linkonce_odr ptr @callee_same_comdat() comdat($comdat_caller) {
+; CHECK-LABEL: define linkonce_odr ptr @callee_same_comdat() comdat($comdat_caller) {
+; CHECK-NEXT:    ret ptr @global3
+;
+  ret ptr @global3
+}
+
+define ptr @comdat_caller() comdat {
+; CHECK-LABEL: define ptr @comdat_caller() comdat {
+; CHECK-NEXT:    ret ptr @global3
+;
+  %res = call ptr @callee_same_comdat()
+  ret ptr %res
+}