[llvm-commits] [llvm] r134516 - in /llvm/trunk: include/llvm/ include/llvm/Support/ include/llvm/Transforms/ lib/CodeGen/ lib/CodeGen/SelectionDAG/ lib/Transforms/Scalar/ lib/Transforms/Utils/ lib/VMCore/ test/CodeGen/Generic/ test/Transforms/Low
Jakub Staszak
jstaszak at apple.com
Wed Jul 6 11:46:15 PDT 2011
Yeah, I don't remember the exact numbers but It looked fine. I will double check that.
- Kuba
On Jul 6, 2011, at 11:33 AM, Eli Friedman wrote:
> On Wed, Jul 6, 2011 at 11:22 AM, Jakub Staszak <jstaszak at apple.com> wrote:
>> Author: kuba
>> Date: Wed Jul 6 13:22:43 2011
>> New Revision: 134516
>>
>> URL: http://llvm.org/viewvc/llvm-project?rev=134516&view=rev
>> Log:
>> Introduce "expect" intrinsic instructions.
>>
>>
>> Added:
>> llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp
>> llvm/trunk/test/CodeGen/Generic/builtin-expect.ll
>> llvm/trunk/test/Transforms/LowerExpectIntrinsic/
>> llvm/trunk/test/Transforms/LowerExpectIntrinsic/basic.ll
>> llvm/trunk/test/Transforms/LowerExpectIntrinsic/dg.exp
>> Modified:
>> llvm/trunk/include/llvm/InitializePasses.h
>> llvm/trunk/include/llvm/Intrinsics.td
>> llvm/trunk/include/llvm/LLVMContext.h
>> llvm/trunk/include/llvm/LinkAllPasses.h
>> llvm/trunk/include/llvm/Support/PassManagerBuilder.h
>> llvm/trunk/include/llvm/Transforms/Scalar.h
>> llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp
>> llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
>> llvm/trunk/lib/Transforms/Scalar/Scalar.cpp
>> llvm/trunk/lib/Transforms/Utils/CMakeLists.txt
>> llvm/trunk/lib/VMCore/LLVMContext.cpp
>>
>> Modified: llvm/trunk/include/llvm/InitializePasses.h
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/InitializePasses.h?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/include/llvm/InitializePasses.h (original)
>> +++ llvm/trunk/include/llvm/InitializePasses.h Wed Jul 6 13:22:43 2011
>> @@ -141,6 +141,7 @@
>> void initializeLoopUnswitchPass(PassRegistry&);
>> void initializeLoopIdiomRecognizePass(PassRegistry&);
>> void initializeLowerAtomicPass(PassRegistry&);
>> +void initializeLowerExpectIntrinsicPass(PassRegistry&);
>> void initializeLowerIntrinsicsPass(PassRegistry&);
>> void initializeLowerInvokePass(PassRegistry&);
>> void initializeLowerSetJmpPass(PassRegistry&);
>>
>> Modified: llvm/trunk/include/llvm/Intrinsics.td
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Intrinsics.td?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/include/llvm/Intrinsics.td (original)
>> +++ llvm/trunk/include/llvm/Intrinsics.td Wed Jul 6 13:22:43 2011
>> @@ -266,6 +266,11 @@
>> [IntrNoMem]>,
>> GCCBuiltin<"__builtin_object_size">;
>>
>> +//===------------------------- Expect Intrinsics --------------------------===//
>> +//
>> +def int_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
>> + LLVMMatchType<0>], [IntrNoMem]>;
>> +
>> //===-------------------- Bit Manipulation Intrinsics ---------------------===//
>> //
>>
>>
>> Modified: llvm/trunk/include/llvm/LLVMContext.h
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LLVMContext.h?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/include/llvm/LLVMContext.h (original)
>> +++ llvm/trunk/include/llvm/LLVMContext.h Wed Jul 6 13:22:43 2011
>> @@ -39,7 +39,8 @@
>> // compile-time performance optimization, not a correctness optimization.
>> enum {
>> MD_dbg = 0, // "dbg"
>> - MD_tbaa = 1 // "tbaa"
>> + MD_tbaa = 1, // "tbaa"
>> + MD_prof = 2 // "prof"
>> };
>>
>> /// getMDKindID - Return a unique non-zero ID for the specified metadata kind.
>>
>> Modified: llvm/trunk/include/llvm/LinkAllPasses.h
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LinkAllPasses.h?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/include/llvm/LinkAllPasses.h (original)
>> +++ llvm/trunk/include/llvm/LinkAllPasses.h Wed Jul 6 13:22:43 2011
>> @@ -92,6 +92,7 @@
>> (void) llvm::createLoopUnswitchPass();
>> (void) llvm::createLoopIdiomPass();
>> (void) llvm::createLoopRotatePass();
>> + (void) llvm::createLowerExpectIntrinsicPass();
>> (void) llvm::createLowerInvokePass();
>> (void) llvm::createLowerSetJmpPass();
>> (void) llvm::createLowerSwitchPass();
>>
>> Modified: llvm/trunk/include/llvm/Support/PassManagerBuilder.h
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Support/PassManagerBuilder.h?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/include/llvm/Support/PassManagerBuilder.h (original)
>> +++ llvm/trunk/include/llvm/Support/PassManagerBuilder.h Wed Jul 6 13:22:43 2011
>> @@ -152,6 +152,7 @@
>> FPM.add(createCFGSimplificationPass());
>> FPM.add(createScalarReplAggregatesPass());
>> FPM.add(createEarlyCSEPass());
>> + FPM.add(createLowerExpectIntrinsicPass());
>> }
>>
>> /// populateModulePassManager - This sets up the primary pass manager.
>>
>> Modified: llvm/trunk/include/llvm/Transforms/Scalar.h
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Scalar.h?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/include/llvm/Transforms/Scalar.h (original)
>> +++ llvm/trunk/include/llvm/Transforms/Scalar.h Wed Jul 6 13:22:43 2011
>> @@ -361,6 +361,14 @@
>> FunctionPass *createInstructionSimplifierPass();
>> extern char &InstructionSimplifierID;
>>
>> +
>> +//===----------------------------------------------------------------------===//
>> +//
>> +// LowerExpectIntriniscs - Removes llvm.expect intrinsics and creates
>> +// "block_weights" metadata.
>> +FunctionPass *createLowerExpectIntrinsicPass();
>> +
>> +
>> } // End llvm namespace
>>
>> #endif
>>
>> Modified: llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp (original)
>> +++ llvm/trunk/lib/CodeGen/IntrinsicLowering.cpp Wed Jul 6 13:22:43 2011
>> @@ -353,6 +353,13 @@
>> report_fatal_error("Code generator does not support intrinsic function '"+
>> Callee->getName()+"'!");
>>
>> + case Intrinsic::expect: {
>> + // Just replace __builtin_expect(exp, c) with EXP.
>> + Value *V = CI->getArgOperand(0);
>> + CI->replaceAllUsesWith(V);
>> + break;
>> + }
>> +
>> // The setjmp/longjmp intrinsics should only exist in the code if it was
>> // never optimized (ie, right out of the CFE), or if it has been hacked on
>> // by the lowerinvoke pass. In both cases, the right thing to do is to
>>
>> Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (original)
>> +++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp Wed Jul 6 13:22:43 2011
>> @@ -4771,6 +4771,13 @@
>> case Intrinsic::flt_rounds:
>> setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, dl, MVT::i32));
>> return 0;
>> +
>> + case Intrinsic::expect: {
>> + // Just replace __builtin_expect(exp, c) with EXP.
>> + setValue(&I, getValue(I.getArgOperand(0)));
>> + return 0;
>> + }
>> +
>> case Intrinsic::trap: {
>> StringRef TrapFuncName = getTrapFunctionName();
>> if (TrapFuncName.empty()) {
>>
>> Modified: llvm/trunk/lib/Transforms/Scalar/Scalar.cpp
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Scalar/Scalar.cpp?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/lib/Transforms/Scalar/Scalar.cpp (original)
>> +++ llvm/trunk/lib/Transforms/Scalar/Scalar.cpp Wed Jul 6 13:22:43 2011
>> @@ -48,6 +48,7 @@
>> initializeLoopUnswitchPass(Registry);
>> initializeLoopIdiomRecognizePass(Registry);
>> initializeLowerAtomicPass(Registry);
>> + initializeLowerExpectIntrinsicPass(Registry);
>> initializeMemCpyOptPass(Registry);
>> initializeObjCARCAliasAnalysisPass(Registry);
>> initializeObjCARCExpandPass(Registry);
>>
>> Modified: llvm/trunk/lib/Transforms/Utils/CMakeLists.txt
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/CMakeLists.txt?rev=134516&r1=134515&r2=134516&view=diff
>> ==============================================================================
>> --- llvm/trunk/lib/Transforms/Utils/CMakeLists.txt (original)
>> +++ llvm/trunk/lib/Transforms/Utils/CMakeLists.txt Wed Jul 6 13:22:43 2011
>> @@ -14,6 +14,7 @@
>> Local.cpp
>> LoopSimplify.cpp
>> LoopUnroll.cpp
>> + LowerExpectIntrinsic.cpp
>> LowerInvoke.cpp
>> LowerSwitch.cpp
>> Mem2Reg.cpp
>>
>> Added: llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp?rev=134516&view=auto
>> ==============================================================================
>> --- llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp (added)
>> +++ llvm/trunk/lib/Transforms/Utils/LowerExpectIntrinsic.cpp Wed Jul 6 13:22:43 2011
>> @@ -0,0 +1,163 @@
>> +#define DEBUG_TYPE "lower-expect-intrinsic"
>> +#include "llvm/Constants.h"
>> +#include "llvm/Function.h"
>> +#include "llvm/BasicBlock.h"
>> +#include "llvm/LLVMContext.h"
>> +#include "llvm/Instructions.h"
>> +#include "llvm/Intrinsics.h"
>> +#include "llvm/Metadata.h"
>> +#include "llvm/Pass.h"
>> +#include "llvm/Transforms/Scalar.h"
>> +#include "llvm/Support/CommandLine.h"
>> +#include "llvm/Support/Debug.h"
>> +#include "llvm/ADT/Statistic.h"
>> +#include <vector>
>> +
>> +using namespace llvm;
>> +
>> +STATISTIC(IfHandled, "Number of 'expect' intrinsic intructions handled");
>> +
>> +static cl::opt<uint32_t>
>> +LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64),
>> + cl::desc("Weight of the branch likely to be taken (default = 64)"));
>> +static cl::opt<uint32_t>
>> +UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4),
>> + cl::desc("Weight of the branch unlikely to be taken (default = 4)"));
>> +
>> +namespace {
>> +
>> + class LowerExpectIntrinsic : public FunctionPass {
>> +
>> + bool HandleSwitchExpect(SwitchInst *SI);
>> +
>> + bool HandleIfExpect(BranchInst *BI);
>> +
>> + public:
>> + static char ID;
>> + LowerExpectIntrinsic() : FunctionPass(ID) {
>> + initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry());
>> + }
>> +
>> + bool runOnFunction(Function &F);
>> + };
>> +}
>> +
>> +
>> +bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) {
>> + CallInst *CI = dyn_cast<CallInst>(SI->getCondition());
>> + if (!CI)
>> + return false;
>> +
>> + Function *Fn = CI->getCalledFunction();
>> + if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
>> + return false;
>> +
>> + Value *ArgValue = CI->getArgOperand(0);
>> + ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
>> + if (!ExpectedValue)
>> + return false;
>> +
>> + LLVMContext &Context = CI->getContext();
>> + const Type *Int32Ty = Type::getInt32Ty(Context);
>> +
>> + unsigned caseNo = SI->findCaseValue(ExpectedValue);
>> + std::vector<Value *> Vec;
>> + unsigned n = SI->getNumCases();
>> + Vec.resize(n + 1); // +1 for MDString
>> +
>> + Vec[0] = MDString::get(Context, "branch_weights");
>> + for (unsigned i = 0; i < n; ++i) {
>> + Vec[i + 1] = ConstantInt::get(Int32Ty, i == caseNo ? LikelyBranchWeight : UnlikelyBranchWeight);
>> + }
>> +
>> + MDNode *WeightsNode = llvm::MDNode::get(Context, Vec);
>> + SI->setMetadata(LLVMContext::MD_prof, WeightsNode);
>> +
>> + SI->setCondition(ArgValue);
>> + return true;
>> +}
>> +
>> +
>> +bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) {
>> + if (BI->isUnconditional())
>> + return false;
>> +
>> + // Handle non-optimized IR code like:
>> + // %expval = call i64 @llvm.expect.i64.i64(i64 %conv1, i64 1)
>> + // %tobool = icmp ne i64 %expval, 0
>> + // br i1 %tobool, label %if.then, label %if.end
>> +
>> + ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition());
>> + if (!CmpI || CmpI->getPredicate() != CmpInst::ICMP_NE)
>> + return false;
>> +
>> + CallInst *CI = dyn_cast<CallInst>(CmpI->getOperand(0));
>> + if (!CI)
>> + return false;
>> +
>> + Function *Fn = CI->getCalledFunction();
>> + if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
>> + return false;
>> +
>> + Value *ArgValue = CI->getArgOperand(0);
>> + ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
>> + if (!ExpectedValue)
>> + return false;
>> +
>> + LLVMContext &Context = CI->getContext();
>> + const Type *Int32Ty = Type::getInt32Ty(Context);
>> + bool Likely = ExpectedValue->isOne();
>> +
>> + // If expect value is equal to 1 it means that we are more likely to take
>> + // branch 0, in other case more likely is branch 1.
>> + Value *Ops[] = {
>> + MDString::get(Context, "branch_weights"),
>> + ConstantInt::get(Int32Ty, Likely ? LikelyBranchWeight : UnlikelyBranchWeight),
>> + ConstantInt::get(Int32Ty, Likely ? UnlikelyBranchWeight : LikelyBranchWeight)
>> + };
>> +
>> + MDNode *WeightsNode = MDNode::get(Context, ArrayRef<Value *>(Ops, 3));
>> + BI->setMetadata(LLVMContext::MD_prof, WeightsNode);
>> +
>> + CmpI->setOperand(0, ArgValue);
>> + return true;
>> +}
>> +
>> +
>> +bool LowerExpectIntrinsic::runOnFunction(Function &F) {
>> + for (Function::iterator I = F.begin(), E = F.end(); I != E;) {
>> + BasicBlock *BB = I++;
>> +
>> + // Create "block_weights" metadata.
>> + if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) {
>> + if (HandleIfExpect(BI))
>> + IfHandled++;
>> + } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
>> + if (HandleSwitchExpect(SI))
>> + IfHandled++;
>> + }
>> +
>> + // remove llvm.expect intrinsics.
>> + for (BasicBlock::iterator BI = BB->begin(), BE = BB->end();
>> + BI != BE; ) {
>> + CallInst *CI = dyn_cast<CallInst>(BI++);
>> + if (!CI)
>> + continue;
>> +
>> + Function *Fn = CI->getCalledFunction();
>> + if (Fn && Fn->getIntrinsicID() == Intrinsic::expect)
>> + CI->eraseFromParent();
>> + }
>> + }
>
> These loops look relatively expensive... have you done benchmarking here?
>
> -Eli
More information about the llvm-commits
mailing list