[llvm] [Inliner] Don't apply the cold-callsite threshold in non-callable functions (PR #211255)

[Spencer Bryngelson via llvm-commits]

https://github.com/sbryngelson updated https://github.com/llvm/llvm-project/pull/211255

>From 48b5eab52db49c2d26abd853f94c9d070057f6b7 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson at gmail.com>
Date: Wed, 22 Jul 2026 08:15:27 -0500
Subject: [PATCH] [Inliner] Don't apply the cold-callsite threshold in
 non-callable functions

A function whose calling convention is not callable is a hardware entry
point, e.g. a GPU kernel. Register allocation there is whole-function and
occupancy is set by the worst case over it, so a call left out of line
costs the hot path too, however cold the call itself is.

Skip the reduced cold-callsite threshold in that case, keyed off the
existing CallingConv::isCallableCC rather than a new target hook.
---
 llvm/lib/Analysis/InlineCost.cpp              |  8 ++-
 .../Inline/AMDGPU/cold-callsite-in-kernel.ll  | 59 +++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/Inline/AMDGPU/cold-callsite-in-kernel.ll

diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index 73f7f5e6cba04..266422d5cd3aa 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -2152,7 +2152,13 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) {
       // behavior to prevent inlining of hot callsites during ThinLTO
       // compile phase.
       Threshold = *HotCallSiteThreshold;
-    } else if (isColdCallSite(Call, CallerBFI)) {
+    } else if (isColdCallSite(Call, CallerBFI) &&
+               isCallableCC(Caller->getCallingConv())) {
+      // In a function that is a hardware entry point rather than something
+      // callable, e.g. a GPU kernel, register allocation is whole-function and
+      // occupancy is set by the worst case over it. A call left out of line
+      // there costs the hot path too, however cold the call itself is, so the
+      // reduced threshold does not apply.
       LLVM_DEBUG(dbgs() << "Cold callsite.\n");
       // Do not apply bonuses for a cold callsite including the
       // LastCallToStatic bonus. While this bonus might result in code size
diff --git a/llvm/test/Transforms/Inline/AMDGPU/cold-callsite-in-kernel.ll b/llvm/test/Transforms/Inline/AMDGPU/cold-callsite-in-kernel.ll
new file mode 100644
index 0000000000000..3f4ba6271058d
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/cold-callsite-in-kernel.ll
@@ -0,0 +1,59 @@
+; RUN: opt -passes=inline -mtriple=amdgpu-amd-amdhsa -inline-instr-cost=50 \
+; RUN:     -pass-remarks=inline -pass-remarks-missed=inline < %s 2>&1 | FileCheck %s
+
+; A call left out of line in a kernel is register allocated against the worst
+; case for the whole kernel, so it costs the hot path too and the reduced
+; cold-callsite threshold does not apply. It still applies in a callable
+; caller. Both callsites are equally cold and the callee has two uses, so the
+; only difference is the caller's calling convention.
+
+; CHECK-DAG: 'callee' inlined into 'kernel'
+; CHECK-DAG: 'callee' not inlined into 'func' because too costly to inline
+
+define internal void @callee(ptr addrspace(1) %p, i32 %x) {
+entry:
+  %v0 = mul i32 %x, 3
+  %v1 = mul i32 %v0, 4
+  %v2 = mul i32 %v1, 5
+  %v3 = mul i32 %v2, 6
+  %v4 = mul i32 %v3, 7
+  %v5 = mul i32 %v4, 8
+  %v6 = mul i32 %v5, 9
+  %v7 = mul i32 %v6, 10
+  %v8 = mul i32 %v7, 11
+  %v9 = mul i32 %v8, 12
+  %v10 = mul i32 %v9, 13
+  %v11 = mul i32 %v10, 14
+  %v12 = mul i32 %v11, 15
+  %v13 = mul i32 %v12, 16
+  %v14 = mul i32 %v13, 17
+  %v15 = mul i32 %v14, 18
+  store i32 %v15, ptr addrspace(1) %p, align 4
+  ret void
+}
+
+define amdgpu_kernel void @kernel(ptr addrspace(1) %p, i32 %x, i1 %c) {
+entry:
+  br i1 %c, label %cold, label %exit, !prof !0
+
+cold:
+  call void @callee(ptr addrspace(1) %p, i32 %x)
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @func(ptr addrspace(1) %p, i32 %x, i1 %c) {
+entry:
+  br i1 %c, label %cold, label %exit, !prof !0
+
+cold:
+  call void @callee(ptr addrspace(1) %p, i32 %x)
+  br label %exit
+
+exit:
+  ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 4000}