[llvm] StackProtector: Use RuntimeLibcalls to query libcall names (PR #147913)

[Matt Arsenault via llvm-commits]

https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/147913

>From 9ae77cd9c0cbe7088357eb36c1fb7bbbc5853d11 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 10 Jul 2025 14:03:11 +0900
Subject: [PATCH] StackProtector: Use RuntimeLibcalls to query libcall names

The compiler should not introduce calls to arbitrary strings
that aren't defined in RuntimeLibcalls. Previously OpenBSD was
disabling the default __stack_chk_fail, but there was no record
of the alternative __stack_smash_handler function it emits instead.

This also avoids a random triple check in the pass.
---
 llvm/include/llvm/IR/RuntimeLibcalls.td       |  4 +++
 llvm/lib/CodeGen/StackProtector.cpp           | 28 ++++++++++++-------
 llvm/lib/IR/RuntimeLibcalls.cpp               |  1 +
 .../NVPTX/no-stack-protector-libcall-error.ll | 10 +++++++
 4 files changed, 33 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/no-stack-protector-libcall-error.ll

diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td
index 267ca6f653eab..077b493dea590 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.td
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.td
@@ -357,6 +357,7 @@ defset list<RuntimeLibcall> LibCalls__OutOfLineAtomic = {
 
 // Stack Protector Fail
 def STACKPROTECTOR_CHECK_FAIL : RuntimeLibcall;
+def STACK_SMASH_HANDLER : RuntimeLibcall;
 
 // Deoptimization
 def DEOPTIMIZE : RuntimeLibcall;
@@ -987,6 +988,9 @@ def bzero : RuntimeLibcallImpl<BZERO>;
 def __bzero : RuntimeLibcallImpl<BZERO>;
 def _Unwind_SjLj_Resume : RuntimeLibcallImpl<UNWIND_RESUME>;
 
+// Used on OpenBSD
+def __stack_smash_handler : RuntimeLibcallImpl<STACK_SMASH_HANDLER>;
+
 def __riscv_flush_icache : RuntimeLibcallImpl<RISCV_FLUSH_ICACHE>;
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp
index 5f866eea7d4e7..3ec70083b7043 100644
--- a/llvm/lib/CodeGen/StackProtector.cpp
+++ b/llvm/lib/CodeGen/StackProtector.cpp
@@ -76,7 +76,7 @@ static bool InsertStackProtectors(const TargetMachine *TM, Function *F,
 
 /// CreateFailBB - Create a basic block to jump to when the stack protector
 /// check fails.
-static BasicBlock *CreateFailBB(Function *F, const Triple &Trip);
+static BasicBlock *CreateFailBB(Function *F, const TargetLowering &TLI);
 
 bool SSPLayoutInfo::shouldEmitSDCheck(const BasicBlock &BB) const {
   return HasPrologue && !HasIRCheck && isa<ReturnInst>(BB.getTerminator());
@@ -673,7 +673,7 @@ bool InsertStackProtectors(const TargetMachine *TM, Function *F,
       // merge pass will merge together all of the various BB into one including
       // fail BB generated by the stack protector pseudo instruction.
       if (!FailBB)
-        FailBB = CreateFailBB(F, TM->getTargetTriple());
+        FailBB = CreateFailBB(F, *TLI);
 
       IRBuilder<> B(CheckLoc);
       Value *Guard = getStackGuard(TLI, M, B);
@@ -706,7 +706,7 @@ bool InsertStackProtectors(const TargetMachine *TM, Function *F,
   return HasPrologue;
 }
 
-BasicBlock *CreateFailBB(Function *F, const Triple &Trip) {
+BasicBlock *CreateFailBB(Function *F, const TargetLowering &TLI) {
   auto *M = F->getParent();
   LLVMContext &Context = F->getContext();
   BasicBlock *FailBB = BasicBlock::Create(Context, "CallStackCheckFailBlk", F);
@@ -716,17 +716,25 @@ BasicBlock *CreateFailBB(Function *F, const Triple &Trip) {
         DILocation::get(Context, 0, 0, F->getSubprogram()));
   FunctionCallee StackChkFail;
   SmallVector<Value *, 1> Args;
-  if (Trip.isOSOpenBSD()) {
-    StackChkFail = M->getOrInsertFunction("__stack_smash_handler",
-                                          Type::getVoidTy(Context),
+
+  if (const char *ChkFailName =
+          TLI.getLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL)) {
+    StackChkFail =
+        M->getOrInsertFunction(ChkFailName, Type::getVoidTy(Context));
+  } else if (const char *SSHName =
+                 TLI.getLibcallName(RTLIB::STACK_SMASH_HANDLER)) {
+    StackChkFail = M->getOrInsertFunction(SSHName, Type::getVoidTy(Context),
                                           PointerType::getUnqual(Context));
     Args.push_back(B.CreateGlobalString(F->getName(), "SSH"));
   } else {
-    StackChkFail =
-        M->getOrInsertFunction("__stack_chk_fail", Type::getVoidTy(Context));
+    Context.emitError("no libcall available for stack protector");
   }
-  cast<Function>(StackChkFail.getCallee())->addFnAttr(Attribute::NoReturn);
-  B.CreateCall(StackChkFail, Args);
+
+  if (StackChkFail) {
+    cast<Function>(StackChkFail.getCallee())->addFnAttr(Attribute::NoReturn);
+    B.CreateCall(StackChkFail, Args);
+  }
+
   B.CreateUnreachable();
   return FailBB;
 }
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 84c67010ca716..6624e24cc7cf5 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -100,6 +100,7 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
 
   if (TT.isOSOpenBSD()) {
     setLibcallImpl(RTLIB::STACKPROTECTOR_CHECK_FAIL, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::STACK_SMASH_HANDLER, RTLIB::__stack_smash_handler);
   }
 
   // Skip default manual processing for targets that have been fully ported to
diff --git a/llvm/test/CodeGen/NVPTX/no-stack-protector-libcall-error.ll b/llvm/test/CodeGen/NVPTX/no-stack-protector-libcall-error.ll
new file mode 100644
index 0000000000000..f877d95dd3769
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/no-stack-protector-libcall-error.ll
@@ -0,0 +1,10 @@
+; RUN: not opt -disable-output -mtriple=nvptx64-- -enable-selectiondag-sp=0 -passes=stack-protector %s 2>&1 | FileCheck %s
+
+; CHECK: error: no libcall available for stack protector
+define void @func() sspreq nounwind {
+  %alloca = alloca i32, align 4
+  call void @capture(ptr %alloca)
+  ret void
+}
+
+declare void @capture(ptr)