Support OpenBSD's native frame protection conventions

[Matthew Dempsky]

OpenBSD's stack smashing protection differs slightly from other
platforms:

  1. The smash handler function is "__stack_smash_handler(const char
     *funcname)" instead of "__stack_chk_fail(void)". 

  2. There's a hidden "long __guard_local" object that gets linked
     into each executable and DSO.

The patch below seems to work for me on OpenBSD/amd64.


Index: lib/CodeGen/StackProtector.cpp
===================================================================
--- lib/CodeGen/StackProtector.cpp	(revision 182911)
+++ lib/CodeGen/StackProtector.cpp	(working copy)
@@ -25,6 +25,8 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
@@ -283,6 +285,11 @@
 
         StackGuardVar = ConstantExpr::getIntToPtr(OffsetVal,
                                       PointerType::get(PtrTy, AddressSpace));
+      } else if (Triple(TLI->getTargetMachine().getTargetTriple()).getOS() ==
+                 llvm::Triple::OpenBSD) {
+        StackGuardVar = M->getOrInsertGlobal("__guard_local", PtrTy);
+        dyn_cast<GlobalValue>(StackGuardVar)
+            ->setVisibility(GlobalValue::HiddenVisibility);
       } else {
         StackGuardVar = M->getOrInsertGlobal("__stack_chk_guard", PtrTy);
       }
@@ -359,12 +366,31 @@
 /// CreateFailBB - Create a basic block to jump to when the stack protector
 /// check fails.
 BasicBlock *StackProtector::CreateFailBB() {
-  BasicBlock *FailBB = BasicBlock::Create(F->getContext(),
-                                          "CallStackCheckFailBlk", F);
-  Constant *StackChkFail =
-    M->getOrInsertFunction("__stack_chk_fail",
-                           Type::getVoidTy(F->getContext()), NULL);
-  CallInst::Create(StackChkFail, "", FailBB);
-  new UnreachableInst(F->getContext(), FailBB);
+  LLVMContext &Context = F->getContext();
+  BasicBlock *FailBB = BasicBlock::Create(Context, "CallStackCheckFailBlk", F);
+  if (Triple(TLI->getTargetMachine().getTargetTriple()).getOS() ==
+      llvm::Triple::OpenBSD) {
+    Constant *StackChkFail = M->getOrInsertFunction(
+        "__stack_smash_handler", Type::getVoidTy(Context),
+        Type::getInt8PtrTy(Context), NULL);
+    Constant *FuncName = new GlobalVariable(
+        *M, ArrayType::get(Type::getInt8Ty(Context), F->getName().size() + 1),
+        true, GlobalVariable::PrivateLinkage,
+        ConstantDataArray::getString(Context, F->getName()), "SSH");
+
+    SmallVector<Constant *, 2> IdxList;
+    IdxList.push_back(ConstantInt::get(Type::getInt8Ty(Context), 0));
+    IdxList.push_back(ConstantInt::get(Type::getInt8Ty(Context), 0));
+
+    SmallVector<Value *, 1> Args;
+    Args.push_back(ConstantExpr::getGetElementPtr(FuncName, IdxList));
+
+    CallInst::Create(StackChkFail, Args, "", FailBB);
+  } else {
+    Constant *StackChkFail = M->getOrInsertFunction(
+        "__stack_chk_fail", Type::getVoidTy(Context), NULL);
+    CallInst::Create(StackChkFail, "", FailBB);
+  }
+  new UnreachableInst(Context, FailBB);
   return FailBB;
 }
Index: test/CodeGen/X86/stack-protector.ll
===================================================================
--- test/CodeGen/X86/stack-protector.ll	(revision 182911)
+++ test/CodeGen/X86/stack-protector.ll	(working copy)
@@ -2,6 +2,7 @@
 ; RUN: llc -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck --check-prefix=LINUX-X64 %s
 ; RUN: llc -code-model=kernel -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck --check-prefix=LINUX-KERNEL-X64 %s
 ; RUN: llc -mtriple=x86_64-apple-darwin < %s -o - | FileCheck --check-prefix=DARWIN-X64 %s
+; RUN: llc -mtriple=amd64-pc-openbsd < %s -o - | FileCheck --check-prefix=OPENBSD-AMD64 %s
 
 %struct.foo = type { [16 x i8] }
 %struct.foo.0 = type { [4 x i8] }
@@ -69,6 +70,10 @@
 ; DARWIN-X64: test1b:
 ; DARWIN-X64: mov{{l|q}} ___stack_chk_guard
 ; DARWIN-X64: callq ___stack_chk_fail
+
+; OPENBSD-AMD64: test1b:
+; OPENBSD-AMD64: movq __guard_local(%rip)
+; OPENBSD-AMD64: callq __stack_smash_handler
   %a.addr = alloca i8*, align 8
   %buf = alloca [16 x i8], align 16
   store i8* %a, i8** %a.addr, align 8