[llvm] [NVPTX] Do not permit calls to ptx_kernel CC (PR #190434)

[Joseph Huber via llvm-commits]

https://github.com/jhuber6 created https://github.com/llvm/llvm-project/pull/190434

Summary:
Does what it says on the tin. I had to delete the uselists portion of
the test because the `.bc` file cannot be regenerated. It was created
with a very old version of LLVM. Either we remove the test entirely,
somehow get llvm-as3.2, or remove this line.


>From 8b03ada97caff703b3e4234055c10c62e9336209 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Fri, 3 Apr 2026 19:00:16 -0500
Subject: [PATCH] [NVPTX] Do not permit calls to ptx_kernel CC

Summary:
Does what it says on the tin. I had to delete the uselists portion of
the test because the `.bc` file cannot be regenerated. It was created
with a very old version of LLVM. Either we remove the test entirely,
somehow get llvm-as3.2, or remove this line.
---
 llvm/include/llvm/IR/CallingConv.h            |  1 +
 llvm/test/Bitcode/calling-conventions.3.2.ll  |  1 -
 .../call-to-non-callable-functions.ll         | 48 +++++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h
index bbf9f8486f31c..249c512dda532 100644
--- a/llvm/include/llvm/IR/CallingConv.h
+++ b/llvm/include/llvm/IR/CallingConv.h
@@ -323,6 +323,7 @@ constexpr bool isCallableCC(CallingConv::ID CC) {
   case CallingConv::AMDGPU_PS:
   case CallingConv::AMDGPU_VS:
   case CallingConv::SPIR_KERNEL:
+  case CallingConv::PTX_Kernel:
     return false;
   default:
     return true;
diff --git a/llvm/test/Bitcode/calling-conventions.3.2.ll b/llvm/test/Bitcode/calling-conventions.3.2.ll
index 678886777ad4c..66d76dd1a93ab 100644
--- a/llvm/test/Bitcode/calling-conventions.3.2.ll
+++ b/llvm/test/Bitcode/calling-conventions.3.2.ll
@@ -1,5 +1,4 @@
 ; RUN:  llvm-dis < %s.bc| FileCheck %s
-; RUN:  verify-uselistorder < %s.bc
 
 ; calling-conventions.3.2.ll.bc was generated by passing this file to llvm-as-3.2.
 ; The test checks that LLVM does not silently misread calling conventions of
diff --git a/llvm/test/Verifier/call-to-non-callable-functions.ll b/llvm/test/Verifier/call-to-non-callable-functions.ll
index f33682c0dbe23..adbb4e90d11ad 100644
--- a/llvm/test/Verifier/call-to-non-callable-functions.ll
+++ b/llvm/test/Verifier/call-to-non-callable-functions.ll
@@ -527,3 +527,51 @@ cont:
 unwind:
   ret void
 }
+
+declare ptx_kernel void @callee_ptx_kernel()
+
+; CHECK: calling convention does not permit calls
+; CHECK-NEXT: call ptx_kernel void @callee_ptx_kernel()
+define ptx_kernel void @call_caller_ptx_kernel() {
+entry:
+  call ptx_kernel void @callee_ptx_kernel()
+  ret void
+}
+
+; CHECK: calling convention does not permit calls
+; CHECK-NEXT: call ptx_kernel void %func()
+define ptx_kernel void @indirect_call_caller_ptx_kernel(ptr %func) {
+entry:
+  call ptx_kernel void %func()
+  ret void
+}
+
+; CHECK: calling convention does not permit calls
+; CHECK-NEXT: invoke ptx_kernel void @callee_ptx_kernel()
+; CHECK-NEXT: to label %cont unwind label %unwind
+define ptx_kernel void @invoke_caller_ptx_kernel() {
+entry:
+  invoke ptx_kernel void @callee_ptx_kernel() to label %cont unwind label %unwind
+  ret void
+
+cont:
+  ret void
+
+unwind:
+  ret void
+}
+
+; CHECK: calling convention does not permit calls
+; CHECK: invoke ptx_kernel void %func()
+; CHECK-NEXT: to label %cont unwind label %unwind
+define ptx_kernel void @indirect_invoke_caller_ptx_kernel(ptr %func) {
+entry:
+  invoke ptx_kernel void %func() to label %cont unwind label %unwind
+  ret void
+
+cont:
+  ret void
+
+unwind:
+  ret void
+}