[llvm-commits] [llvm] r134516 - in /llvm/trunk: include/llvm/ include/llvm/Support/ include/llvm/Transforms/ lib/CodeGen/ lib/CodeGen/SelectionDAG/ lib/Transforms/Scalar/ lib/Transforms/Utils/ lib/VMCore/ test/CodeGen/Generic/ test/Transforms/Low
Eli Friedman
eli.friedman at gmail.com
Wed Jul 6 11:33:37 PDT 2011
On Wed, Jul 6, 2011 at 11:22 AM, Jakub Staszak <jstaszak at apple.com> wrote:
> Author: kuba
> Date: Wed Jul 6 13:22:43 2011
> New Revision: 134516
>
> URL: http://llvm.org/viewvc/llvm-project?rev=134516&view=rev
> Log:
> Introduce "expect" intrinsic instructions.
>
>
> Added:
> llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp
> llvm/trunk/test/CodeGen/Generic/builtin-expect.ll
> llvm/trunk/test/Transforms/LowerExpectIntrinsic/
> llvm/trunk/test/Transforms/LowerExpectIntrinsic/basic.ll
> llvm/trunk/test/Transforms/LowerExpectIntrinsic/dg.exp
> Modified:
> llvm/trunk/include/llvm/InitializePasses.h
> llvm/trunk/include/llvm/Intrinsics.td
> llvm/trunk/include/llvm/LLVMContext.h
> llvm/trunk/include/llvm/LinkAllPasses.h
> llvm/trunk/include/llvm/Support/PassManagerBuilder.h
> llvm/trunk/include/llvm/Transforms/Scalar.h
> llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp
> llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
> llvm/trunk/lib/Transforms/Scalar/Scalar.cpp
> llvm/trunk/lib/Transforms/Utils/CMakeLists.txt
> llvm/trunk/lib/VMCore/LLVMContext.cpp
>
> Modified: llvm/trunk/include/llvm/InitializePasses.h
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/InitializePasses.h?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/include/llvm/InitializePasses.h (original)
> +++ llvm/trunk/include/llvm/InitializePasses.h Wed Jul 6 13:22:43 2011
> @@ -141,6 +141,7 @@
> void initializeLoopUnswitchPass(PassRegistry&);
> void initializeLoopIdiomRecognizePass(PassRegistry&);
> void initializeLowerAtomicPass(PassRegistry&);
> +void initializeLowerExpectIntrinsicPass(PassRegistry&);
> void initializeLowerIntrinsicsPass(PassRegistry&);
> void initializeLowerInvokePass(PassRegistry&);
> void initializeLowerSetJmpPass(PassRegistry&);
>
> Modified: llvm/trunk/include/llvm/Intrinsics.td
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Intrinsics.td?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/include/llvm/Intrinsics.td (original)
> +++ llvm/trunk/include/llvm/Intrinsics.td Wed Jul 6 13:22:43 2011
> @@ -266,6 +266,11 @@
> [IntrNoMem]>,
> GCCBuiltin<"__builtin_object_size">;
>
> +//===------------------------- Expect Intrinsics --------------------------===//
> +//
> +def int_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
> + LLVMMatchType<0>], [IntrNoMem]>;
> +
> //===-------------------- Bit Manipulation Intrinsics ---------------------===//
> //
>
>
> Modified: llvm/trunk/include/llvm/LLVMContext.h
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LLVMContext.h?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/include/llvm/LLVMContext.h (original)
> +++ llvm/trunk/include/llvm/LLVMContext.h Wed Jul 6 13:22:43 2011
> @@ -39,7 +39,8 @@
> // compile-time performance optimization, not a correctness optimization.
> enum {
> MD_dbg = 0, // "dbg"
> - MD_tbaa = 1 // "tbaa"
> + MD_tbaa = 1, // "tbaa"
> + MD_prof = 2 // "prof"
> };
>
> /// getMDKindID - Return a unique non-zero ID for the specified metadata kind.
>
> Modified: llvm/trunk/include/llvm/LinkAllPasses.h
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LinkAllPasses.h?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/include/llvm/LinkAllPasses.h (original)
> +++ llvm/trunk/include/llvm/LinkAllPasses.h Wed Jul 6 13:22:43 2011
> @@ -92,6 +92,7 @@
> (void) llvm::createLoopUnswitchPass();
> (void) llvm::createLoopIdiomPass();
> (void) llvm::createLoopRotatePass();
> + (void) llvm::createLowerExpectIntrinsicPass();
> (void) llvm::createLowerInvokePass();
> (void) llvm::createLowerSetJmpPass();
> (void) llvm::createLowerSwitchPass();
>
> Modified: llvm/trunk/include/llvm/Support/PassManagerBuilder.h
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Support/PassManagerBuilder.h?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/include/llvm/Support/PassManagerBuilder.h (original)
> +++ llvm/trunk/include/llvm/Support/PassManagerBuilder.h Wed Jul 6 13:22:43 2011
> @@ -152,6 +152,7 @@
> FPM.add(createCFGSimplificationPass());
> FPM.add(createScalarReplAggregatesPass());
> FPM.add(createEarlyCSEPass());
> + FPM.add(createLowerExpectIntrinsicPass());
> }
>
> /// populateModulePassManager - This sets up the primary pass manager.
>
> Modified: llvm/trunk/include/llvm/Transforms/Scalar.h
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Scalar.h?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/include/llvm/Transforms/Scalar.h (original)
> +++ llvm/trunk/include/llvm/Transforms/Scalar.h Wed Jul 6 13:22:43 2011
> @@ -361,6 +361,14 @@
> FunctionPass *createInstructionSimplifierPass();
> extern char &InstructionSimplifierID;
>
> +
> +//===----------------------------------------------------------------------===//
> +//
> +// LowerExpectIntriniscs - Removes llvm.expect intrinsics and creates
> +// "block_weights" metadata.
> +FunctionPass *createLowerExpectIntrinsicPass();
> +
> +
> } // End llvm namespace
>
> #endif
>
> Modified: llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp (original)
> +++ llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp Wed Jul 6 13:22:43 2011
> @@ -353,6 +353,13 @@
> report_fatal_error("Code generator does not support intrinsic function '"+
> Callee->getName()+"'!");
>
> + case Intrinsic::expect: {
> + // Just replace __builtin_expect(exp, c) with EXP.
> + Value *V = CI->getArgOperand(0);
> + CI->replaceAllUsesWith(V);
> + break;
> + }
> +
> // The setjmp/longjmp intrinsics should only exist in the code if it was
> // never optimized (ie, right out of the CFE), or if it has been hacked on
> // by the lowerinvoke pass. In both cases, the right thing to do is to
>
> Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (original)
> +++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp Wed Jul 6 13:22:43 2011
> @@ -4771,6 +4771,13 @@
> case Intrinsic::flt_rounds:
> setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, dl, MVT::i32));
> return 0;
> +
> + case Intrinsic::expect: {
> + // Just replace __builtin_expect(exp, c) with EXP.
> + setValue(&I, getValue(I.getArgOperand(0)));
> + return 0;
> + }
> +
> case Intrinsic::trap: {
> StringRef TrapFuncName = getTrapFunctionName();
> if (TrapFuncName.empty()) {
>
> Modified: llvm/trunk/lib/Transforms/Scalar/Scalar.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Scalar/Scalar.cpp?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/lib/Transforms/Scalar/Scalar.cpp (original)
> +++ llvm/trunk/lib/Transforms/Scalar/Scalar.cpp Wed Jul 6 13:22:43 2011
> @@ -48,6 +48,7 @@
> initializeLoopUnswitchPass(Registry);
> initializeLoopIdiomRecognizePass(Registry);
> initializeLowerAtomicPass(Registry);
> + initializeLowerExpectIntrinsicPass(Registry);
> initializeMemCpyOptPass(Registry);
> initializeObjCARCAliasAnalysisPass(Registry);
> initializeObjCARCExpandPass(Registry);
>
> Modified: llvm/trunk/lib/Transforms/Utils/CMakeLists.txt
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/CMakeLists.txt?rev=134516&r1=134515&r2=134516&view=diff
> ==============================================================================
> --- llvm/trunk/lib/Transforms/Utils/CMakeLists.txt (original)
> +++ llvm/trunk/lib/Transforms/Utils/CMakeLists.txt Wed Jul 6 13:22:43 2011
> @@ -14,6 +14,7 @@
> Local.cpp
> LoopSimplify.cpp
> LoopUnroll.cpp
> + LowerExpectIntrinsic.cpp
> LowerInvoke.cpp
> LowerSwitch.cpp
> Mem2Reg.cpp
>
> Added: llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp?rev=134516&view=auto
> ==============================================================================
> --- llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp (added)
> +++ llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp Wed Jul 6 13:22:43 2011
> @@ -0,0 +1,163 @@
> +#define DEBUG_TYPE "lower-expect-intrinsic"
> +#include "llvm/Constants.h"
> +#include "llvm/Function.h"
> +#include "llvm/BasicBlock.h"
> +#include "llvm/LLVMContext.h"
> +#include "llvm/Instructions.h"
> +#include "llvm/Intrinsics.h"
> +#include "llvm/Metadata.h"
> +#include "llvm/Pass.h"
> +#include "llvm/Transforms/Scalar.h"
> +#include "llvm/Support/CommandLine.h"
> +#include "llvm/Support/Debug.h"
> +#include "llvm/ADT/Statistic.h"
> +#include <vector>
> +
> +using namespace llvm;
> +
> +STATISTIC(IfHandled, "Number of 'expect' intrinsic intructions handled");
> +
> +static cl::opt<uint32_t>
> +LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64),
> + cl::desc("Weight of the branch likely to be taken (default = 64)"));
> +static cl::opt<uint32_t>
> +UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4),
> + cl::desc("Weight of the branch unlikely to be taken (default = 4)"));
> +
> +namespace {
> +
> + class LowerExpectIntrinsic : public FunctionPass {
> +
> + bool HandleSwitchExpect(SwitchInst *SI);
> +
> + bool HandleIfExpect(BranchInst *BI);
> +
> + public:
> + static char ID;
> + LowerExpectIntrinsic() : FunctionPass(ID) {
> + initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry());
> + }
> +
> + bool runOnFunction(Function &F);
> + };
> +}
> +
> +
> +bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) {
> + CallInst *CI = dyn_cast<CallInst>(SI->getCondition());
> + if (!CI)
> + return false;
> +
> + Function *Fn = CI->getCalledFunction();
> + if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
> + return false;
> +
> + Value *ArgValue = CI->getArgOperand(0);
> + ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
> + if (!ExpectedValue)
> + return false;
> +
> + LLVMContext &Context = CI->getContext();
> + const Type *Int32Ty = Type::getInt32Ty(Context);
> +
> + unsigned caseNo = SI->findCaseValue(ExpectedValue);
> + std::vector<Value *> Vec;
> + unsigned n = SI->getNumCases();
> + Vec.resize(n + 1); // +1 for MDString
> +
> + Vec[0] = MDString::get(Context, "branch_weights");
> + for (unsigned i = 0; i < n; ++i) {
> + Vec[i + 1] = ConstantInt::get(Int32Ty, i == caseNo ? LikelyBranchWeight : UnlikelyBranchWeight);
> + }
> +
> + MDNode *WeightsNode = llvm::MDNode::get(Context, Vec);
> + SI->setMetadata(LLVMContext::MD_prof, WeightsNode);
> +
> + SI->setCondition(ArgValue);
> + return true;
> +}
> +
> +
> +bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) {
> + if (BI->isUnconditional())
> + return false;
> +
> + // Handle non-optimized IR code like:
> + // %expval = call i64 @llvm.expect.i64.i64(i64 %conv1, i64 1)
> + // %tobool = icmp ne i64 %expval, 0
> + // br i1 %tobool, label %if.then, label %if.end
> +
> + ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition());
> + if (!CmpI || CmpI->getPredicate() != CmpInst::ICMP_NE)
> + return false;
> +
> + CallInst *CI = dyn_cast<CallInst>(CmpI->getOperand(0));
> + if (!CI)
> + return false;
> +
> + Function *Fn = CI->getCalledFunction();
> + if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
> + return false;
> +
> + Value *ArgValue = CI->getArgOperand(0);
> + ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
> + if (!ExpectedValue)
> + return false;
> +
> + LLVMContext &Context = CI->getContext();
> + const Type *Int32Ty = Type::getInt32Ty(Context);
> + bool Likely = ExpectedValue->isOne();
> +
> + // If expect value is equal to 1 it means that we are more likely to take
> + // branch 0, in other case more likely is branch 1.
> + Value *Ops[] = {
> + MDString::get(Context, "branch_weights"),
> + ConstantInt::get(Int32Ty, Likely ? LikelyBranchWeight : UnlikelyBranchWeight),
> + ConstantInt::get(Int32Ty, Likely ? UnlikelyBranchWeight : LikelyBranchWeight)
> + };
> +
> + MDNode *WeightsNode = MDNode::get(Context, ArrayRef<Value *>(Ops, 3));
> + BI->setMetadata(LLVMContext::MD_prof, WeightsNode);
> +
> + CmpI->setOperand(0, ArgValue);
> + return true;
> +}
> +
> +
> +bool LowerExpectIntrinsic::runOnFunction(Function &F) {
> + for (Function::iterator I = F.begin(), E = F.end(); I != E;) {
> + BasicBlock *BB = I++;
> +
> + // Create "block_weights" metadata.
> + if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) {
> + if (HandleIfExpect(BI))
> + IfHandled++;
> + } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
> + if (HandleSwitchExpect(SI))
> + IfHandled++;
> + }
> +
> + // remove llvm.expect intrinsics.
> + for (BasicBlock::iterator BI = BB->begin(), BE = BB->end();
> + BI != BE; ) {
> + CallInst *CI = dyn_cast<CallInst>(BI++);
> + if (!CI)
> + continue;
> +
> + Function *Fn = CI->getCalledFunction();
> + if (Fn && Fn->getIntrinsicID() == Intrinsic::expect)
> + CI->eraseFromParent();
> + }
> + }
These loops look relatively expensive... have you done benchmarking here?
-Eli
More information about the llvm-commits
mailing list