Support OpenBSD's native frame protection conventions

[Duncan Sands]

Hi Matthew,

On 31/05/13 22:49, Matthew Dempsky wrote:
> OpenBSD's stack smashing protection differs slightly from other
> platforms:
>
>    1. The smash handler function is "__stack_smash_handler(const char
>       *funcname)" instead of "__stack_chk_fail(void)".
>
>    2. There's a hidden "long __guard_local" object that gets linked
>       into each executable and DSO.
>
> The patch below seems to work for me on OpenBSD/amd64.
>
>
> Index: lib/CodeGen/StackProtector.cpp
> ===================================================================
> --- lib/CodeGen/StackProtector.cpp	(revision 182911)
> +++ lib/CodeGen/StackProtector.cpp	(working copy)
> @@ -25,6 +25,8 @@
>   #include "llvm/IR/DataLayout.h"
>   #include "llvm/IR/DerivedTypes.h"
>   #include "llvm/IR/Function.h"
> +#include "llvm/IR/GlobalValue.h"
> +#include "llvm/IR/GlobalVariable.h"
>   #include "llvm/IR/Instructions.h"
>   #include "llvm/IR/Intrinsics.h"
>   #include "llvm/IR/Module.h"
> @@ -283,6 +285,11 @@
>
>           StackGuardVar = ConstantExpr::getIntToPtr(OffsetVal,
>                                         PointerType::get(PtrTy, AddressSpace));
> +      } else if (Triple(TLI->getTargetMachine().getTargetTriple()).getOS() ==
> +                 llvm::Triple::OpenBSD) {

maybe you could store the triple in the pass object, since a couple of places
need it.

> +        StackGuardVar = M->getOrInsertGlobal("__guard_local", PtrTy);
> +        dyn_cast<GlobalValue>(StackGuardVar)

This can be cast rather than dyn_cast.

> +            ->setVisibility(GlobalValue::HiddenVisibility);
>         } else {
>           StackGuardVar = M->getOrInsertGlobal("__stack_chk_guard", PtrTy);
>         }
> @@ -359,12 +366,31 @@
>   /// CreateFailBB - Create a basic block to jump to when the stack protector
>   /// check fails.
>   BasicBlock *StackProtector::CreateFailBB() {
> -  BasicBlock *FailBB = BasicBlock::Create(F->getContext(),
> -                                          "CallStackCheckFailBlk", F);
> -  Constant *StackChkFail =
> -    M->getOrInsertFunction("__stack_chk_fail",
> -                           Type::getVoidTy(F->getContext()), NULL);
> -  CallInst::Create(StackChkFail, "", FailBB);
> -  new UnreachableInst(F->getContext(), FailBB);
> +  LLVMContext &Context = F->getContext();
> +  BasicBlock *FailBB = BasicBlock::Create(Context, "CallStackCheckFailBlk", F);
> +  if (Triple(TLI->getTargetMachine().getTargetTriple()).getOS() ==
> +      llvm::Triple::OpenBSD) {
> +    Constant *StackChkFail = M->getOrInsertFunction(
> +        "__stack_smash_handler", Type::getVoidTy(Context),
> +        Type::getInt8PtrTy(Context), NULL);
> +    Constant *FuncName = new GlobalVariable(
> +        *M, ArrayType::get(Type::getInt8Ty(Context), F->getName().size() + 1),
> +        true, GlobalVariable::PrivateLinkage,
> +        ConstantDataArray::getString(Context, F->getName()), "SSH");

Here you could first construct the constant string
   Constant *NameStr = ConstantDataArray::getString(Context, F->getName());
and then use its type when constructing the global:
   Constant *FuncName = new GlobalVariable(*M, NameStr->getType(), true,
GlobalVariable::PrivateLinkage, NameStr, "SSH");

Otherwise looks OK to me.

Ciao, Duncan.

> +
> +    SmallVector<Constant *, 2> IdxList;
> +    IdxList.push_back(ConstantInt::get(Type::getInt8Ty(Context), 0));
> +    IdxList.push_back(ConstantInt::get(Type::getInt8Ty(Context), 0));
> +
> +    SmallVector<Value *, 1> Args;
> +    Args.push_back(ConstantExpr::getGetElementPtr(FuncName, IdxList));
> +
> +    CallInst::Create(StackChkFail, Args, "", FailBB);
> +  } else {
> +    Constant *StackChkFail = M->getOrInsertFunction(
> +        "__stack_chk_fail", Type::getVoidTy(Context), NULL);
> +    CallInst::Create(StackChkFail, "", FailBB);
> +  }
> +  new UnreachableInst(Context, FailBB);
>     return FailBB;
>   }
> Index: test/CodeGen/X86/stack-protector.ll
> ===================================================================
> --- test/CodeGen/X86/stack-protector.ll	(revision 182911)
> +++ test/CodeGen/X86/stack-protector.ll	(working copy)
> @@ -2,6 +2,7 @@
>   ; RUN: llc -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck --check-prefix=LINUX-X64 %s
>   ; RUN: llc -code-model=kernel -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck --check-prefix=LINUX-KERNEL-X64 %s
>   ; RUN: llc -mtriple=x86_64-apple-darwin < %s -o - | FileCheck --check-prefix=DARWIN-X64 %s
> +; RUN: llc -mtriple=amd64-pc-openbsd < %s -o - | FileCheck --check-prefix=OPENBSD-AMD64 %s
>
>   %struct.foo = type { [16 x i8] }
>   %struct.foo.0 = type { [4 x i8] }
> @@ -69,6 +70,10 @@
>   ; DARWIN-X64: test1b:
>   ; DARWIN-X64: mov{{l|q}} ___stack_chk_guard
>   ; DARWIN-X64: callq ___stack_chk_fail
> +
> +; OPENBSD-AMD64: test1b:
> +; OPENBSD-AMD64: movq __guard_local(%rip)
> +; OPENBSD-AMD64: callq __stack_smash_handler
>     %a.addr = alloca i8*, align 8
>     %buf = alloca [16 x i8], align 16
>     store i8* %a, i8** %a.addr, align 8
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits
>