[PATCH] An intrinsic and lowering for instrumentation based profiling

[Duncan P. N. Exon Smith]

> On 2014-Dec-04, at 15:50, Justin Bogner <mail at justinbogner.com> wrote:
> 
> This introduces the ``llvm.instrprof_increment`` intrinsic and the
> ``-instrprof`` pass. These provide the infrastructure for writing
> counters for profiling, as in clang's ``-fprofile-instr-generate``.
> 
> The implementation of the instrprof pass is ported directly out of the
> CodeGenPGO classes in clang, and with the followup in clang that rips
> that code out to use these new intrinsics this ends up being NFC.
> 
> Doing the instrumentation this way opens some doors in terms of
> improving the counter performance. For example, this will make it
> simple to experiment with alternate lowering strategies, and allows us
> to try handling profiling specially in some optimizations if we want
> to.
> 
> Finally, this drastically simplifies the frontend and puts all of the
> lowering logic in one place.
> 
> <instrprof-intrinsic.patch>

> diff --git a/docs/LangRef.rst b/docs/LangRef.rst
> index 3412ed3..68e4d65 100644
> --- a/docs/LangRef.rst
> +++ b/docs/LangRef.rst
> @@ -7492,6 +7492,50 @@ time library.
>  This instrinsic does *not* empty the instruction pipeline. Modifications
>  of the current function are outside the scope of the intrinsic.
>  
> +'``llvm.instrprof_increment``' Intrinsic
> +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> +
> +Syntax:
> +"""""""
> +
> +::
> +
> +      declare void @llvm.instrprof_increment(i8* <name>, i32 <num-counters>,
> +                                             i64 <hash>, i64 <counter-index>)

The types seem off: you have an `i32` for the number of counters but an
`i64` for the index into them.  I think `i32` makes sense for both.

Bike-shed: I think it'd be clearer to reorder the arguments so the
counter index and number of counters are adjacent.  I'd prefer either of:

    (i8* <name>, i64 <hash>, i32 <counter-index>, i32 <num-counters>)
    (i8* <name>, i64 <hash>, i32 <num-counters>,  i32 <counter-index>)

> +
> +Overview:
> +"""""""""
> +
> +The '``llvm.instrprof_increment``' intrinsic can be emitted by a
> +frontend for use with instrumentation based profiling. These will be
> +lowered by the ``-instrprof`` pass to generate execution counts of a
> +program at runtime.
> +
> +Arguments:
> +""""""""""
> +
> +The first argument is a pointer to a global variable containing the
> +name of the entity being instrumented. This should generally be the
> +(mangled) function name for a set of counters.
> +
> +The second and third arguments are the number of counters in this set
> +and a hash value that can be used by the consumer of the profile data
> +to detect changes to the instrumented source. It is an error if
> +``num-counters`` or ``hash`` differ between two instances of
> +``instrprof_increment`` that refer to the same name.
> +
> +The last argument refers to which of the counters for ``name`` should
> +be incremented. It should be a value between 0 and ``num-counters``.
> +
> +Semantics:
> +""""""""""
> +
> +This intrinsic represents an increment of a profiling counter. It will
> +cause the ``-instrprof`` pass to generate the appropriate data
> +structures and the code to increment the appropriate value, in a
> +format that can be written out by a compiler runtime and consumed via
> +the ``llvm-profdata`` tool.
> +
>  Standard C Library Intrinsics
>  -----------------------------
>  
> diff --git a/include/llvm/IR/IntrinsicInst.h b/include/llvm/IR/IntrinsicInst.h
> index e3d7999..dbdc210 100644
> --- a/include/llvm/IR/IntrinsicInst.h
> +++ b/include/llvm/IR/IntrinsicInst.h
> @@ -322,6 +322,33 @@ namespace llvm {
>      Value *getSrc() const { return const_cast<Value*>(getArgOperand(1)); }
>    };
>  
> +  /// This represents the llvm.instrprof_increment intrinsic.
> +  class InstrProfIncrementInst : public IntrinsicInst {
> +  public:
> +    static inline bool classof(const IntrinsicInst *I) {
> +      return I->getIntrinsicID() == Intrinsic::instrprof_increment;
> +    }
> +    static inline bool classof(const Value *V) {
> +      return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
> +    }
> +
> +    GlobalVariable *getName() const {
> +      return cast<GlobalVariable>(
> +          const_cast<Value *>(getArgOperand(0))->stripPointerCasts());
> +    }
> +
> +    ConstantInt *getNumCounters() const {
> +      return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1)));
> +    }
> +
> +    ConstantInt *getHash() const {
> +      return cast<ConstantInt>(const_cast<Value *>(getArgOperand(2)));
> +    }
> +
> +    ConstantInt *getIndex() const {
> +      return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3)));
> +    }
> +  };
>  }
>  
>  #endif
> diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td
> index 5457e95..9d59be1 100644
> --- a/include/llvm/IR/Intrinsics.td
> +++ b/include/llvm/IR/Intrinsics.td
> @@ -287,6 +287,12 @@ def int_stackprotector : Intrinsic<[], [llvm_ptr_ty, llvm_ptrptr_ty], []>;
>  def int_stackprotectorcheck : Intrinsic<[], [llvm_ptrptr_ty],
>                                          [IntrReadWriteArgMem]>;
>  
> +// A counter increment for instrumentation based profiling.
> +def int_instrprof_increment : Intrinsic<[],
> +                                        [llvm_ptr_ty, llvm_i32_ty,
> +                                         llvm_i64_ty, llvm_i64_ty],
> +                                        []>;
> +
>  //===------------------- Standard C Library Intrinsics --------------------===//
>  //
>  
> diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h
> index edce794..2b99e1b 100644
> --- a/include/llvm/InitializePasses.h
> +++ b/include/llvm/InitializePasses.h
> @@ -121,6 +121,7 @@ void initializeEarlyIfConverterPass(PassRegistry&);
>  void initializeEdgeBundlesPass(PassRegistry&);
>  void initializeExpandPostRAPass(PassRegistry&);
>  void initializeGCOVProfilerPass(PassRegistry&);
> +void initializeInstrProfilingPass(PassRegistry&);
>  void initializeAddressSanitizerPass(PassRegistry&);
>  void initializeAddressSanitizerModulePass(PassRegistry&);
>  void initializeMemorySanitizerPass(PassRegistry&);
> diff --git a/include/llvm/LinkAllPasses.h b/include/llvm/LinkAllPasses.h
> index 2c16c8c..fad3865 100644
> --- a/include/llvm/LinkAllPasses.h
> +++ b/include/llvm/LinkAllPasses.h
> @@ -78,6 +78,7 @@ namespace {
>        (void) llvm::createDomOnlyViewerPass();
>        (void) llvm::createDomViewerPass();
>        (void) llvm::createGCOVProfilerPass();
> +      (void) llvm::createInstrProfilingPass();
>        (void) llvm::createFunctionInliningPass();
>        (void) llvm::createAlwaysInlinerPass();
>        (void) llvm::createGlobalDCEPass();
> diff --git a/include/llvm/Transforms/Instrumentation.h b/include/llvm/Transforms/Instrumentation.h
> index 4737cb3..24e3ef7 100644
> --- a/include/llvm/Transforms/Instrumentation.h
> +++ b/include/llvm/Transforms/Instrumentation.h
> @@ -63,6 +63,18 @@ struct GCOVOptions {
>  ModulePass *createGCOVProfilerPass(const GCOVOptions &Options =
>                                     GCOVOptions::getDefault());
>  
> +/// Options for the frontend instrumentation based profiling pass.
> +struct InstrProfOptions {
> +  InstrProfOptions() : NoRedZone(false) {}
> +
> +  // Add the 'noredzone' attribute to added runtime library calls.
> +  bool NoRedZone;
> +};
> +
> +/// Insert frontend instrumentation based profiling.
> +ModulePass *createInstrProfilingPass(
> +    const InstrProfOptions &Options = InstrProfOptions());
> +
>  // Insert AddressSanitizer (address sanity checking) instrumentation
>  FunctionPass *createAddressSanitizerFunctionPass();
>  ModulePass *createAddressSanitizerModulePass();
> diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
> index eeecbf1..35473d2 100644
> --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
> +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
> @@ -5477,6 +5477,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
>      visitGCRelocate(I);
>      return nullptr;
>    }
> +  case Intrinsic::instrprof_increment:
> +    llvm_unreachable("instrprof failed to lower an increment");
>    }
>  }
>  
> diff --git a/lib/Transforms/Instrumentation/CMakeLists.txt b/lib/Transforms/Instrumentation/CMakeLists.txt
> index a6a3106..92e1091 100644
> --- a/lib/Transforms/Instrumentation/CMakeLists.txt
> +++ b/lib/Transforms/Instrumentation/CMakeLists.txt
> @@ -5,6 +5,7 @@ add_llvm_library(LLVMInstrumentation
>    GCOVProfiling.cpp
>    MemorySanitizer.cpp
>    Instrumentation.cpp
> +  InstrProfiling.cpp
>    SanitizerCoverage.cpp
>    ThreadSanitizer.cpp
>    )
> diff --git a/lib/Transforms/Instrumentation/InstrProfiling.cpp b/lib/Transforms/Instrumentation/InstrProfiling.cpp
> new file mode 100644
> index 0000000..6c9cc26
> --- /dev/null
> +++ b/lib/Transforms/Instrumentation/InstrProfiling.cpp
> @@ -0,0 +1,313 @@
> +//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
> +//
> +//                     The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +//
> +// This pass lowers instrprof_increment intrinsics emitted by a frontend for
> +// profiling. It also builds the data structures and initialization code needed
> +// for updating execution counts and emitting the profile at runtime.
> +//
> +//===----------------------------------------------------------------------===//
> +
> +#include "llvm/Transforms/Instrumentation.h"
> +
> +#include "llvm/ADT/Triple.h"
> +#include "llvm/IR/IRBuilder.h"
> +#include "llvm/IR/IntrinsicInst.h"
> +#include "llvm/IR/Module.h"
> +#include "llvm/Transforms/Utils/ModuleUtils.h"
> +
> +using namespace llvm;
> +
> +#define DEBUG_TYPE "instrprof"
> +
> +namespace {
> +
> +class InstrProfiling : public ModulePass {
> +public:
> +  static char ID;
> +
> +  InstrProfiling() : ModulePass(ID) {}
> +
> +  InstrProfiling(const InstrProfOptions &Options)
> +      : ModulePass(ID), Options(Options) {}
> +
> +  const char *getPassName() const override {
> +    return "Frontend instrumentation-based coverage lowering";
> +  }
> +
> +  bool runOnModule(Module &M) override;
> +
> +  void getAnalysisUsage(AnalysisUsage &AU) const override {
> +    AU.setPreservesCFG();
> +  }
> +
> +private:
> +  InstrProfOptions Options;
> +  Module *M;
> +  DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
> +  std::vector<Value *> UsedVars;
> +
> +  bool isMachO() const {
> +    return Triple(M->getTargetTriple()).isOSBinFormatMachO();
> +  }
> +
> +  /// Get the section name for the counter variables.
> +  StringRef getCountersSection() const {
> +    return isMachO() ? "__DATA,__llvm_prf_cnts" : "__llvm_prf_cnts";
> +  }
> +
> +  /// Get the section name for the name variables.
> +  StringRef getNameSection() const {
> +    return isMachO() ? "__DATA,__llvm_prf_names" : "__llvm_prf_names";
> +  }
> +
> +  /// Get the section name for the profile data variables.
> +  StringRef getDataSection() const {
> +    return isMachO() ? "__DATA,__llvm_prf_data" : "__llvm_prf_data";
> +  }
> +
> +  /// Replace instrprof_increment with an increment of the appropriate value.
> +  void lowerIncrement(InstrProfIncrementInst *Inc);
> +
> +  /// Get the region counters for an increment, creating them if necessary.
> +  ///
> +  /// If the counter array doesn't yet exist, the profile data variables
> +  /// referring to them will also be created.
> +  GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
> +
> +  /// Emit runtime registration functions for each profile data variable.
> +  bool emitRegistration();
> +
> +  /// Emit the necessary plumbing to pull in the runtime initialization.
> +  bool emitRuntimeHook();
> +
> +  /// Add uses of our data variables and runtime hook.
> +  bool emitUses();
> +
> +  /// Create a static initializer for our data, on platforms that need it.
> +  bool emitInitialization();
> +};
> +
> +} // anonymous namespace
> +
> +char InstrProfiling::ID = 0;
> +INITIALIZE_PASS(InstrProfiling, "instrprof",
> +                "Frontend instrumentation-based coverage lowering.", false,
> +                false)
> +
> +ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) {
> +  return new InstrProfiling(Options);
> +}
> +
> +bool InstrProfiling::runOnModule(Module &M) {
> +  bool MadeChange = false;
> +
> +  this->M = &M;
> +  RegionCounters.clear();
> +  UsedVars.clear();
> +
> +  for (Function &F : M)
> +    for (BasicBlock &BB : F)
> +      for (auto I = BB.begin(), E = BB.end(); I != E;)
> +        if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
> +          lowerIncrement(Inc);
> +          MadeChange = true;
> +        }
> +  MadeChange |= emitRegistration();
> +  MadeChange |= emitRuntimeHook();
> +  MadeChange |= emitUses();
> +  MadeChange |= emitInitialization();
> +  return MadeChange;

Why isn't this the following?

    if (!MadeChange)
      return false;

    emitRegistration();
    emitRuntimeHook();
    emitUses();
    emitInitialization();
    return true;

In particular, should the pass emit anything if there are no counters?
Why?

Can you add a test for when the pass gets run but there aren't any
counters?  (What you're testing for depends on the above...)

> +}
> +
> +void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
> +  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
> +
> +  IRBuilder<> Builder(Inc->getParent(), *Inc);
> +  uint64_t Index = Inc->getIndex()->getZExtValue();
> +  llvm::Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
> +  llvm::Value *Count = Builder.CreateLoad(Addr, "pgocount");
> +  Count = Builder.CreateAdd(Count, Builder.getInt64(1));
> +  Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
> +  Inc->eraseFromParent();
> +}
> +
> +/// Get the name of a profiling variable for a particular function.
> +static std::string getVarName(InstrProfIncrementInst *Inc, StringRef VarName) {
> +  auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer());
> +  StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
> +  return ("__llvm_profile_" + VarName + "_" + Name).str();
> +}
> +
> +GlobalVariable *
> +InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
> +  GlobalVariable *Name = Inc->getName();
> +  auto It = RegionCounters.find(Name);
> +  if (It != RegionCounters.end())
> +    return It->second;
> +
> +  // Move the name variable to the right section.
> +  Name->setSection(getNameSection());
> +  Name->setAlignment(1);
> +
> +  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
> +  LLVMContext &Ctx = M->getContext();
> +  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
> +
> +  // Create the counters variable.
> +  auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
> +                                      Constant::getNullValue(CounterTy),
> +                                      getVarName(Inc, "counters"));
> +  Counters->setVisibility(Name->getVisibility());
> +  Counters->setSection(getCountersSection());
> +  Counters->setAlignment(8);
> +
> +  RegionCounters[Inc->getName()] = Counters;
> +
> +  // Create data variable.
> +  auto *NameArrayTy = Name->getType()->getPointerElementType();
> +  auto *Int32Ty = Type::getInt32Ty(Ctx);
> +  auto *Int64Ty = Type::getInt64Ty(Ctx);
> +  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
> +  auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
> +
> +  Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
> +  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
> +  Constant *DataVals[] = {
> +      ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
> +      ConstantInt::get(Int32Ty, NumCounters),
> +      ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
> +      ConstantExpr::getBitCast(Name, Int8PtrTy),
> +      ConstantExpr::getBitCast(Counters, Int64PtrTy)};
> +  auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
> +                                  ConstantStruct::get(DataTy, DataVals),
> +                                  getVarName(Inc, "data"));
> +  Data->setVisibility(Name->getVisibility());
> +  Data->setSection(getDataSection());
> +  Data->setAlignment(8);
> +
> +  // Mark the data variable as used so that it isn't stripped out.
> +  UsedVars.push_back(Data);
> +
> +  return Counters;
> +}
> +
> +bool InstrProfiling::emitRegistration() {
> +  // Don't do this for Darwin.  compiler-rt uses linker magic.
> +  if (Triple(M->getTargetTriple()).isOSDarwin())
> +    return false;
> +  if (UsedVars.empty())
> +    return false;
> +
> +  // Construct the function.
> +  auto *VoidTy = Type::getVoidTy(M->getContext());
> +  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
> +  auto *RegisterFTy = FunctionType::get(VoidTy, false);
> +  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
> +                                     "__llvm_profile_register_functions", M);
> +  RegisterF->setUnnamedAddr(true);
> +  if (Options.NoRedZone)
> +    RegisterF->addFnAttr(Attribute::NoRedZone);
> +
> +  auto *RuntimeRegisterTy = llvm::FunctionType::get(VoidTy, VoidPtrTy, false);
> +  auto *RuntimeRegisterF =
> +      Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
> +                       "__llvm_profile_register_function", M);
> +
> +  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
> +  for (Value *Data : UsedVars)
> +    IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
> +  IRB.CreateRetVoid();
> +  return true;
> +}
> +
> +bool InstrProfiling::emitRuntimeHook() {
> +  const char *const RuntimeVarName = "__llvm_profile_runtime";
> +  const char *const RuntimeUserName = "__llvm_profile_runtime_user";
> +
> +  // If the module's provided its own runtime, we don't need to do anything.
> +  if (M->getGlobalVariable(RuntimeVarName))
> +    return false;
> +
> +  // Declare an external variable that will pull in the runtime initialization.
> +  auto *Int32Ty = Type::getInt32Ty(M->getContext());
> +  auto *Var =
> +      new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
> +                         nullptr, RuntimeVarName);
> +
> +  // Make a function that uses it.
> +  auto *User =
> +      Function::Create(FunctionType::get(Int32Ty, false),
> +                       GlobalValue::LinkOnceODRLinkage, RuntimeUserName, M);
> +  User->addFnAttr(Attribute::NoInline);
> +  if (Options.NoRedZone)
> +    User->addFnAttr(Attribute::NoRedZone);
> +
> +  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
> +  auto *Load = IRB.CreateLoad(Var);
> +  IRB.CreateRet(Load);
> +
> +  // Mark the user variable as used so that it isn't stripped out.
> +  UsedVars.push_back(User);
> +
> +  return true;
> +}
> +
> +bool InstrProfiling::emitUses() {
> +  if (UsedVars.empty())
> +    return false;
> +
> +  GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
> +  std::vector<Constant*> MergedVars;
> +  if (LLVMUsed) {
> +    // Collect the existing members of llvm.used.
> +    ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
> +    for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I)
> +      MergedVars.push_back(Inits->getOperand(I));
> +    LLVMUsed->eraseFromParent();
> +  }
> +
> +  Type *i8PTy = Type::getInt8PtrTy(M->getContext());
> +  // Add uses for our data.
> +  for (auto *Value : UsedVars)
> +    MergedVars.push_back(
> +        ConstantExpr::getBitCast(cast<llvm::Constant>(Value), i8PTy));
> +
> +  // Recreate llvm.used.
> +  ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
> +  LLVMUsed = new llvm::GlobalVariable(
> +      *M, ATy, false, llvm::GlobalValue::AppendingLinkage,
> +      llvm::ConstantArray::get(ATy, MergedVars), "llvm.used");
> +
> +  LLVMUsed->setSection("llvm.metadata");

It feels kind of awkward to delete and recreate `@llvm.used`, I'd rather
just reset its initializer.  But since you need to collect its current
members anyway, maybe that's just as awkward...

> +  return true;
> +}
> +
> +bool InstrProfiling::emitInitialization() {
> +  Constant *RegisterF = M->getFunction("__llvm_profile_register_functions");
> +  if (!RegisterF)
> +    return false;
> +
> +  // Create the initialization function.
> +  auto *VoidTy = Type::getVoidTy(M->getContext());
> +  auto *F =
> +      Function::Create(FunctionType::get(VoidTy, false),
> +                       GlobalValue::InternalLinkage, "__llvm_profile_init", M);
> +  F->setUnnamedAddr(true);
> +  F->addFnAttr(Attribute::NoInline);
> +  if (Options.NoRedZone)
> +    F->addFnAttr(Attribute::NoRedZone);
> +
> +  // Add the basic block and the necessary calls.
> +  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
> +  IRB.CreateCall(RegisterF);
> +  IRB.CreateRetVoid();
> +
> +  appendToGlobalCtors(*M, F, 0);
> +  return true;
> +}
> diff --git a/lib/Transforms/Instrumentation/Instrumentation.cpp b/lib/Transforms/Instrumentation/Instrumentation.cpp
> index 8e95367..a91fc0e 100644
> --- a/lib/Transforms/Instrumentation/Instrumentation.cpp
> +++ b/lib/Transforms/Instrumentation/Instrumentation.cpp
> @@ -25,6 +25,7 @@ void llvm::initializeInstrumentation(PassRegistry &Registry) {
>    initializeAddressSanitizerModulePass(Registry);
>    initializeBoundsCheckingPass(Registry);
>    initializeGCOVProfilerPass(Registry);
> +  initializeInstrProfilingPass(Registry);
>    initializeMemorySanitizerPass(Registry);
>    initializeThreadSanitizerPass(Registry);
>    initializeSanitizerCoverageModulePass(Registry);
> diff --git a/test/Instrumentation/InstrProfiling/noruntime.ll b/test/Instrumentation/InstrProfiling/noruntime.ll
> new file mode 100644
> index 0000000..c812495
> --- /dev/null
> +++ b/test/Instrumentation/InstrProfiling/noruntime.ll
> @@ -0,0 +1,16 @@
> +;; Check that we don't emit the runtime hooks if the user provided them.
> +
> +; RUN: opt < %s -instrprof -S | FileCheck %s
> +; CHECK-NOT: define {{.*}} @__llvm_profile_runtime_user()
> +; CHECK-NOT: load i32* @__llvm_profile_runtime
> +
> + at __llvm_profile_runtime = global i32 0, align 4
> +
> + at __llvm_profile_name_foo = hidden constant [3 x i8] c"foo"
> +
> +define void @foo() {
> +  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8]* @__llvm_profile_name_foo, i32 0, i32 0), i32 1, i64 0, i64 0)
> +  ret void
> +}
> +
> +declare void @llvm.instrprof.increment(i8*, i32, i64, i64)
> diff --git a/test/Instrumentation/InstrProfiling/platform.ll b/test/Instrumentation/InstrProfiling/platform.ll
> new file mode 100644
> index 0000000..86dec0b
> --- /dev/null
> +++ b/test/Instrumentation/InstrProfiling/platform.ll
> @@ -0,0 +1,29 @@
> +;; Checks for platform specific section names and initialization code.
> +
> +; RUN: opt < %s -mtriple=x86_64-apple-macosx10.10.0 -instrprof -S | FileCheck %s -check-prefix=MACHO
> +; RUN: opt < %s -mtriple=x86_64-unknown-linux -instrprof -S | FileCheck %s -check-prefix=ELF
> +
> + at __llvm_profile_name_foo = hidden constant [3 x i8] c"foo"
> +; MACHO: @__llvm_profile_name_foo = hidden constant [3 x i8] c"foo", section "__DATA,__llvm_prf_names", align 1
> +; ELF: @__llvm_profile_name_foo = hidden constant [3 x i8] c"foo", section "__llvm_prf_names", align 1
> +
> +; MACHO: @__llvm_profile_counters_foo = hidden global [1 x i64] zeroinitializer, section "__DATA,__llvm_prf_cnts", align 8
> +; ELF: @__llvm_profile_counters_foo = hidden global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", align 8
> +
> +; MACHO: @__llvm_profile_data_foo = hidden constant {{.*}}, section "__DATA,__llvm_prf_data", align 8
> +; ELF: @__llvm_profile_data_foo = hidden constant {{.*}}, section "__llvm_prf_data", align 8
> +define void @foo() {
> +  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8]* @__llvm_profile_name_foo, i32 0, i32 0), i32 1, i64 0, i64 0)
> +  ret void
> +}
> +
> +declare void @llvm.instrprof.increment(i8*, i32, i64, i64)
> +
> +;; Emit registration functions for platforms that don't find the
> +;; symbols by their sections.
> +
> +; MACHO-NOT: define internal void @__llvm_profile_register_functions
> +; ELF: define internal void @__llvm_profile_register_functions
> +
> +; MACHO-NOT: define internal void @__llvm_profile_init
> +; ELF: define internal void @__llvm_profile_init
> diff --git a/test/Instrumentation/InstrProfiling/profiling.ll b/test/Instrumentation/InstrProfiling/profiling.ll
> new file mode 100644
> index 0000000..7a12858
> --- /dev/null
> +++ b/test/Instrumentation/InstrProfiling/profiling.ll
> @@ -0,0 +1,37 @@
> +; RUN: opt < %s -instrprof -S | FileCheck %s
> +
> +target triple = "x86_64-apple-macosx10.10.0"
> +
> + at __llvm_profile_name_foo = hidden constant [3 x i8] c"foo"
> +; CHECK: @__llvm_profile_name_foo = hidden constant [3 x i8] c"foo", section "__DATA,__llvm_prf_names", align 1
> + at __llvm_profile_name_bar = hidden constant [4 x i8] c"bar\00"
> +; CHECK: @__llvm_profile_name_bar = hidden constant [4 x i8] c"bar\00", section "__DATA,__llvm_prf_names", align 1
> + at baz_prof_name = hidden constant [3 x i8] c"baz"
> +; CHECK: @baz_prof_name = hidden constant [3 x i8] c"baz", section "__DATA,__llvm_prf_names", align 1
> +
> +; CHECK: @__llvm_profile_counters_foo = hidden global [1 x i64] zeroinitializer, section "__DATA,__llvm_prf_cnts", align 8
> +; CHECK: @__llvm_profile_data_foo = hidden constant {{.*}}, section "__DATA,__llvm_prf_data", align 8
> +define void @foo() {
> +  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8]* @__llvm_profile_name_foo, i32 0, i32 0), i32 1, i64 0, i64 0)
> +  ret void
> +}
> +
> +; CHECK: @__llvm_profile_counters_bar = hidden global [1 x i64] zeroinitializer, section "__DATA,__llvm_prf_cnts", align 8
> +; CHECK: @__llvm_profile_data_bar = hidden constant {{.*}}, section "__DATA,__llvm_prf_data", align 8
> +define void @bar() {
> +  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([4 x i8]* @__llvm_profile_name_bar, i32 0, i32 0), i32 1, i64 0, i64 0)
> +  ret void
> +}
> +
> +; CHECK: @__llvm_profile_counters_baz = hidden global [3 x i64] zeroinitializer, section "__DATA,__llvm_prf_cnts", align 8
> +; CHECK: @__llvm_profile_data_baz = hidden constant {{.*}}, section "__DATA,__llvm_prf_data", align 8
> +define void @baz() {
> +  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8]* @baz_prof_name, i32 0, i32 0), i32 3, i64 0, i64 0)
> +  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8]* @baz_prof_name, i32 0, i32 0), i32 3, i64 0, i64 1)
> +  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8]* @baz_prof_name, i32 0, i32 0), i32 3, i64 0, i64 2)
> +  ret void
> +}
> +
> +declare void @llvm.instrprof.increment(i8*, i32, i64, i64)
> +
> +; CHECK: @llvm.used = appending global {{.*}} @__llvm_profile_data_foo {{.*}} @__llvm_profile_data_bar {{.*}} @__llvm_profile_data_baz {{.*}} section "llvm.metadata"
>