[clang] [llvm] [HIP] Fix ubsan function checks applied to kernel functions (PR #188872)

[Joseph Huber via llvm-commits]

https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/188872

>From d19ff2504d785353d2f800efc96595e820723717 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Thu, 26 Mar 2026 18:05:57 -0500
Subject: [PATCH 1/3] [HIP] Fix ubsan function checks applied to kernel
 functions

Summary:
The 'function' check requires inserting eight bytes of magic before each
function. The HIP runtime expects and enforces 256 byte alignment. When
the instrumentation inserts the eight bytes this is done after the
alignment, which means that the HIP runtime then points the PC to an
invalid instruction by truncating the address to 256 byte alignment. The
OpenMP runtime doesn't do this.

The purpose of this function is to handle indirect calls, and it's
impossible to indirectly call a kernel anyway, so we should just
suppress this in this case. The only other solution would be to add the
alignment back before we emit the label, but that would be meaningless
because it'd just replace the magic bytes with zeroes.
---
 clang/lib/CodeGen/CodeGenFunction.cpp |  6 ++++--
 clang/test/CodeGen/AMDGPU/sanitizer.c | 20 ++++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/CodeGen/AMDGPU/sanitizer.c

diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 77c6866bbefa6..a9ba4309b04be 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1033,9 +1033,11 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
   }
 
   // If we are checking function types, emit a function type signature as
-  // prologue data.
+  // prologue data. Kernel functions have strict alignment requirements and
+  // cannot be call indirectly so we do not instrument them.
   if (FD && SanOpts.has(SanitizerKind::Function) &&
-      !FD->getType()->isCFIUncheckedCalleeFunctionType()) {
+      !FD->getType()->isCFIUncheckedCalleeFunctionType() &&
+      !Fn->hasKernelCallingConv()) {
     if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
       llvm::LLVMContext &Ctx = Fn->getContext();
       llvm::MDBuilder MDB(Ctx);
diff --git a/clang/test/CodeGen/AMDGPU/sanitizer.c b/clang/test/CodeGen/AMDGPU/sanitizer.c
new file mode 100644
index 0000000000000..3b41e4dd8babe
--- /dev/null
+++ b/clang/test/CodeGen/AMDGPU/sanitizer.c
@@ -0,0 +1,20 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -cc1 -triple amdgcn-amd-amdhsa -emit-llvm -fsanitize=function %s -o - | FileCheck %s
+
+// CHECK-LABEL: define dso_local void @function(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] !func_sanitize [[META2:![0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void function() {}
+
+// CHECK-LABEL: define dso_local amdgpu_kernel void @kernel(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @function() #[[ATTR1:[0-9]+]]
+// CHECK-NEXT:    ret void
+//
+[[clang::device_kernel]] void kernel() { function(); }
+//.
+// CHECK: [[META2]] = !{i32 -1056584962, i32 1717976574}
+//.

>From 1f61f59788ae0931755d9c25122144369d0df06e Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Fri, 3 Apr 2026 11:48:06 -0500
Subject: [PATCH 2/3] Use iscallablecc

---
 clang/lib/CodeGen/CodeGenFunction.cpp | 2 +-
 llvm/include/llvm/IR/CallingConv.h    | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index a9ba4309b04be..b920266b59808 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1037,7 +1037,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
   // cannot be call indirectly so we do not instrument them.
   if (FD && SanOpts.has(SanitizerKind::Function) &&
       !FD->getType()->isCFIUncheckedCalleeFunctionType() &&
-      !Fn->hasKernelCallingConv()) {
+      llvm::isCallableCC(Fn->getCallingConv())) {
     if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
       llvm::LLVMContext &Ctx = Fn->getContext();
       llvm::MDBuilder MDB(Ctx);
diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h
index bbf9f8486f31c..249c512dda532 100644
--- a/llvm/include/llvm/IR/CallingConv.h
+++ b/llvm/include/llvm/IR/CallingConv.h
@@ -323,6 +323,7 @@ constexpr bool isCallableCC(CallingConv::ID CC) {
   case CallingConv::AMDGPU_PS:
   case CallingConv::AMDGPU_VS:
   case CallingConv::SPIR_KERNEL:
+  case CallingConv::PTX_Kernel:
     return false;
   default:
     return true;

>From 6fdd6f11d7438f7fbcaad5c79f6ccfce2ef2ec00 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Fri, 3 Apr 2026 13:48:04 -0500
Subject: [PATCH 3/3] fix test

---
 llvm/test/Bitcode/calling-conventions.3.2.ll | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/llvm/test/Bitcode/calling-conventions.3.2.ll b/llvm/test/Bitcode/calling-conventions.3.2.ll
index 678886777ad4c..a8a9e584954b6 100644
--- a/llvm/test/Bitcode/calling-conventions.3.2.ll
+++ b/llvm/test/Bitcode/calling-conventions.3.2.ll
@@ -1,5 +1,4 @@
 ; RUN:  llvm-dis < %s.bc| FileCheck %s
-; RUN:  verify-uselistorder < %s.bc
 
 ; calling-conventions.3.2.ll.bc was generated by passing this file to llvm-as-3.2.
 ; The test checks that LLVM does not silently misread calling conventions of
@@ -140,12 +139,6 @@ define void @call_msp430_intrcc() {
   ret void
 }
 
-define void @call_ptx_kernel() {
-; CHECK: call ptx_kernel void @ptx_kernel
-  call ptx_kernel void @ptx_kernel()
-  ret void
-}
-
 define void @call_ptx_device() {
 ; CHECK: call ptx_device void @ptx_device
   call ptx_device void @ptx_device()