[llvm] r296028 - Add call branch annotation for ICP promoted direct call in SamplePGO mode.
Dehao Chen via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 23 14:15:19 PST 2017
Author: dehao
Date: Thu Feb 23 16:15:18 2017
New Revision: 296028
URL: http://llvm.org/viewvc/llvm-project?rev=296028&view=rev
Log:
Add call branch annotation for ICP promoted direct call in SamplePGO mode.
Summary: SamplePGO uses branch_weight annotation to represent callsite hotness. When ICP promotes an indirect call to direct call, we need to make sure the direct call is annotated with branch_weight in SamplePGO mode, so that downstream function inliner can use hot callsite heuristic.
Reviewers: davidxl, eraman, xur
Reviewed By: davidxl, xur
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D30282
Modified:
llvm/trunk/include/llvm/Passes/PassBuilder.h
llvm/trunk/include/llvm/Transforms/Instrumentation.h
llvm/trunk/include/llvm/Transforms/PGOInstrumentation.h
llvm/trunk/lib/Passes/PassBuilder.cpp
llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp
llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp
llvm/trunk/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
llvm/trunk/test/Transforms/PGOProfile/indirect_call_promotion.ll
Modified: llvm/trunk/include/llvm/Passes/PassBuilder.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Passes/PassBuilder.h?rev=296028&r1=296027&r2=296028&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Passes/PassBuilder.h (original)
+++ llvm/trunk/include/llvm/Passes/PassBuilder.h Thu Feb 23 16:15:18 2017
@@ -32,6 +32,7 @@ struct PGOOptions {
std::string ProfileGenFile = "";
std::string ProfileUseFile = "";
bool RunProfileGen = false;
+ bool SamplePGO = false;
};
/// \brief This class provides access to building LLVM's passes.
Modified: llvm/trunk/include/llvm/Transforms/Instrumentation.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Instrumentation.h?rev=296028&r1=296027&r2=296028&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Transforms/Instrumentation.h (original)
+++ llvm/trunk/include/llvm/Transforms/Instrumentation.h Thu Feb 23 16:15:18 2017
@@ -86,7 +86,8 @@ ModulePass *createGCOVProfilerPass(const
ModulePass *createPGOInstrumentationGenLegacyPass();
ModulePass *
createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""));
-ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false);
+ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false,
+ bool SamplePGO = false);
// Helper function to check if it is legal to promote indirect call \p Inst
// to a direct call of function \p F. Stores the reason in \p Reason.
@@ -102,9 +103,12 @@ bool isLegalToPromote(Instruction *Inst,
// TotalCount is the profile count value that the instruction executes.
// Count is the profile count value that F is the target function.
// These two values are used to update the branch weight.
+// If \p AttachProfToDirectCall is true, a prof metadata is attached to the
+// new direct call to contain \p Count.
// Returns the promoted direct call instruction.
Instruction *promoteIndirectCall(Instruction *Inst, Function *F, uint64_t Count,
- uint64_t TotalCount);
+ uint64_t TotalCount,
+ bool AttachProfToDirectCall);
/// Options for the frontend instrumentation based profiling pass.
struct InstrProfOptions {
Modified: llvm/trunk/include/llvm/Transforms/PGOInstrumentation.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/PGOInstrumentation.h?rev=296028&r1=296027&r2=296028&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Transforms/PGOInstrumentation.h (original)
+++ llvm/trunk/include/llvm/Transforms/PGOInstrumentation.h Thu Feb 23 16:15:18 2017
@@ -38,10 +38,13 @@ private:
/// The indirect function call promotion pass.
class PGOIndirectCallPromotion : public PassInfoMixin<PGOIndirectCallPromotion> {
public:
- PGOIndirectCallPromotion(bool IsInLTO = false) : InLTO(IsInLTO) {}
+ PGOIndirectCallPromotion(bool IsInLTO = false, bool SamplePGO = false)
+ : InLTO(IsInLTO), SamplePGO(SamplePGO) {}
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+
private:
bool InLTO;
+ bool SamplePGO;
};
} // End llvm namespace
Modified: llvm/trunk/lib/Passes/PassBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Passes/PassBuilder.cpp?rev=296028&r1=296027&r2=296028&view=diff
==============================================================================
--- llvm/trunk/lib/Passes/PassBuilder.cpp (original)
+++ llvm/trunk/lib/Passes/PassBuilder.cpp Thu Feb 23 16:15:18 2017
@@ -486,13 +486,14 @@ PassBuilder::buildPerModuleDefaultPipeli
// Add all the requested passes for PGO Instrumentation, if requested.
if (PGOOpt) {
- assert(PGOOpt->RunProfileGen || !PGOOpt->ProfileUseFile.empty());
+ assert(PGOOpt->RunProfileGen || PGOOpt->SamplePGO ||
+ !PGOOpt->ProfileUseFile.empty());
addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen,
PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile);
}
// Indirect call promotion that promotes intra-module targes only.
- MPM.addPass(PGOIndirectCallPromotion());
+ MPM.addPass(PGOIndirectCallPromotion(false, PGOOpt && PGOOpt->SamplePGO));
// Require the GlobalsAA analysis for the module so we can query it within
// the CGSCC pipeline.
@@ -665,7 +666,8 @@ ModulePassManager PassBuilder::buildLTOD
// left by the earlier promotion pass that promotes intra-module targets.
// This two-step promotion is to save the compile time. For LTO, it should
// produce the same result as if we only do promotion here.
- MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */));
+ MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */,
+ PGOOpt && PGOOpt->SamplePGO));
// Propagate constants at call sites into the functions they call. This
// opens opportunities for globalopt (and inlining) by substituting function
Modified: llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp?rev=296028&r1=296027&r2=296028&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp (original)
+++ llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp Thu Feb 23 16:15:18 2017
@@ -431,7 +431,8 @@ void PassManagerBuilder::populateModuleP
// earlier in the pass pipeline, here before globalopt. Otherwise imported
// available_externally functions look unreferenced and are removed.
if (PerformThinLTO)
- MPM.add(createPGOIndirectCallPromotionLegacyPass(/*InLTO = */ true));
+ MPM.add(createPGOIndirectCallPromotionLegacyPass(/*InLTO = */ true,
+ !PGOSampleUse.empty()));
if (!DisableUnitAtATime) {
// Infer attributes about declarations if possible.
@@ -458,7 +459,8 @@ void PassManagerBuilder::populateModuleP
// Indirect call promotion that promotes intra-module targets only.
// For ThinLTO this is done earlier due to interactions with globalopt
// for imported functions.
- MPM.add(createPGOIndirectCallPromotionLegacyPass());
+ MPM.add(
+ createPGOIndirectCallPromotionLegacyPass(false, !PGOSampleUse.empty()));
}
if (EnableNonLTOGlobalsModRef)
@@ -681,7 +683,8 @@ void PassManagerBuilder::addLTOOptimizat
// left by the earlier promotion pass that promotes intra-module targets.
// This two-step promotion is to save the compile time. For LTO, it should
// produce the same result as if we only do promotion here.
- PM.add(createPGOIndirectCallPromotionLegacyPass(true));
+ PM.add(
+ createPGOIndirectCallPromotionLegacyPass(true, !PGOSampleUse.empty()));
// Propagate constants at call sites into the functions they call. This
// opens opportunities for globalopt (and inlining) by substituting function
Modified: llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp?rev=296028&r1=296027&r2=296028&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp (original)
+++ llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp Thu Feb 23 16:15:18 2017
@@ -646,7 +646,7 @@ bool SampleProfileLoader::inlineHotFunct
// We set the probability to 80% taken to indicate that the static
// call is likely taken.
DI = dyn_cast<Instruction>(
- promoteIndirectCall(I, CalledFunction, 80, 100)
+ promoteIndirectCall(I, CalledFunction, 80, 100, false)
->stripPointerCasts());
PromotedInsns.insert(I);
} else {
Modified: llvm/trunk/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp?rev=296028&r1=296027&r2=296028&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp (original)
+++ llvm/trunk/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp Thu Feb 23 16:15:18 2017
@@ -80,6 +80,12 @@ static cl::opt<bool> ICPLTOMode("icp-lto
cl::desc("Run indirect-call promotion in LTO "
"mode"));
+// Set if the pass is called in SamplePGO mode. The difference for SamplePGO
+// mode is it will add prof metadatato the created direct call.
+static cl::opt<bool>
+ ICPSamplePGOMode("icp-samplepgo", cl::init(false), cl::Hidden,
+ cl::desc("Run indirect-call promotion in SamplePGO mode"));
+
// If the option is set to true, only call instructions will be considered for
// transformation -- invoke instructions will be ignored.
static cl::opt<bool>
@@ -105,8 +111,8 @@ class PGOIndirectCallPromotionLegacyPass
public:
static char ID;
- PGOIndirectCallPromotionLegacyPass(bool InLTO = false)
- : ModulePass(ID), InLTO(InLTO) {
+ PGOIndirectCallPromotionLegacyPass(bool InLTO = false, bool SamplePGO = false)
+ : ModulePass(ID), InLTO(InLTO), SamplePGO(SamplePGO) {
initializePGOIndirectCallPromotionLegacyPassPass(
*PassRegistry::getPassRegistry());
}
@@ -119,6 +125,10 @@ private:
// If this pass is called in LTO. We need to special handling the PGOFuncName
// for the static variables due to LTO's internalization.
bool InLTO;
+
+ // If this pass is called in SamplePGO. We need to add the prof metadata to
+ // the promoted direct call.
+ bool SamplePGO;
};
} // end anonymous namespace
@@ -128,8 +138,9 @@ INITIALIZE_PASS(PGOIndirectCallPromotion
"direct calls.",
false, false)
-ModulePass *llvm::createPGOIndirectCallPromotionLegacyPass(bool InLTO) {
- return new PGOIndirectCallPromotionLegacyPass(InLTO);
+ModulePass *llvm::createPGOIndirectCallPromotionLegacyPass(bool InLTO,
+ bool SamplePGO) {
+ return new PGOIndirectCallPromotionLegacyPass(InLTO, SamplePGO);
}
namespace {
@@ -144,6 +155,8 @@ private:
// defines.
InstrProfSymtab *Symtab;
+ bool SamplePGO;
+
// Test if we can legally promote this direct-call of Target.
bool isPromotionLegal(Instruction *Inst, uint64_t Target, Function *&F,
const char **Reason = nullptr);
@@ -175,9 +188,9 @@ private:
ICallPromotionFunc &operator=(const ICallPromotionFunc &other) = delete;
public:
- ICallPromotionFunc(Function &Func, Module *Modu, InstrProfSymtab *Symtab)
- : F(Func), M(Modu), Symtab(Symtab) {
- }
+ ICallPromotionFunc(Function &Func, Module *Modu, InstrProfSymtab *Symtab,
+ bool SamplePGO)
+ : F(Func), M(Modu), Symtab(Symtab), SamplePGO(SamplePGO) {}
bool processFunction();
};
@@ -509,10 +522,14 @@ static void insertCallRetPHI(Instruction
// Ret = phi(Ret1, Ret2);
// It adds type casts for the args do not match the parameters and the return
// value. Branch weights metadata also updated.
+// If \p AttachProfToDirectCall is true, a prof metadata is attached to the
+// new direct call to contain \p Count. This is used by SamplePGO inliner to
+// check callsite hotness.
// Returns the promoted direct call instruction.
Instruction *llvm::promoteIndirectCall(Instruction *Inst,
Function *DirectCallee, uint64_t Count,
- uint64_t TotalCount) {
+ uint64_t TotalCount,
+ bool AttachProfToDirectCall) {
assert(DirectCallee != nullptr);
BasicBlock *BB = Inst->getParent();
// Just to suppress the non-debug build warning.
@@ -527,6 +544,14 @@ Instruction *llvm::promoteIndirectCall(I
Instruction *NewInst =
createDirectCallInst(Inst, DirectCallee, DirectCallBB, MergeBB);
+ if (AttachProfToDirectCall) {
+ SmallVector<uint32_t, 1> Weights;
+ Weights.push_back(Count);
+ MDBuilder MDB(NewInst->getContext());
+ dyn_cast<Instruction>(NewInst->stripPointerCasts())
+ ->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
+ }
+
// Move Inst from MergeBB to IndirectCallBB.
Inst->removeFromParent();
IndirectCallBB->getInstList().insert(IndirectCallBB->getFirstInsertionPt(),
@@ -569,7 +594,7 @@ uint32_t ICallPromotionFunc::tryToPromot
for (auto &C : Candidates) {
uint64_t Count = C.Count;
- promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount);
+ promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount, SamplePGO);
assert(TotalCount >= Count);
TotalCount -= Count;
NumOfPGOICallPromotion++;
@@ -610,7 +635,7 @@ bool ICallPromotionFunc::processFunction
}
// A wrapper function that does the actual work.
-static bool promoteIndirectCalls(Module &M, bool InLTO) {
+static bool promoteIndirectCalls(Module &M, bool InLTO, bool SamplePGO) {
if (DisableICP)
return false;
InstrProfSymtab Symtab;
@@ -621,7 +646,7 @@ static bool promoteIndirectCalls(Module
continue;
if (F.hasFnAttribute(Attribute::OptimizeNone))
continue;
- ICallPromotionFunc ICallPromotion(F, &M, &Symtab);
+ ICallPromotionFunc ICallPromotion(F, &M, &Symtab, SamplePGO);
bool FuncChanged = ICallPromotion.processFunction();
if (ICPDUMPAFTER && FuncChanged) {
DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs()));
@@ -638,11 +663,14 @@ static bool promoteIndirectCalls(Module
bool PGOIndirectCallPromotionLegacyPass::runOnModule(Module &M) {
// Command-line option has the priority for InLTO.
- return promoteIndirectCalls(M, InLTO | ICPLTOMode);
+ return promoteIndirectCalls(M, InLTO | ICPLTOMode,
+ SamplePGO | ICPSamplePGOMode);
}
-PreservedAnalyses PGOIndirectCallPromotion::run(Module &M, ModuleAnalysisManager &AM) {
- if (!promoteIndirectCalls(M, InLTO | ICPLTOMode))
+PreservedAnalyses PGOIndirectCallPromotion::run(Module &M,
+ ModuleAnalysisManager &AM) {
+ if (!promoteIndirectCalls(M, InLTO | ICPLTOMode,
+ SamplePGO | ICPSamplePGOMode))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
Modified: llvm/trunk/test/Transforms/PGOProfile/indirect_call_promotion.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/PGOProfile/indirect_call_promotion.ll?rev=296028&r1=296027&r2=296028&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/PGOProfile/indirect_call_promotion.ll (original)
+++ llvm/trunk/test/Transforms/PGOProfile/indirect_call_promotion.ll Thu Feb 23 16:15:18 2017
@@ -1,4 +1,6 @@
; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo | FileCheck %s --check-prefix=ICALL-PROM-SAMPLEPGO
; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
; RUN: opt < %s -pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
@@ -40,6 +42,7 @@ entry:
; ICALL-PROM: br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]]
; ICALL-PROM: if.true.direct_targ:
; ICALL-PROM: [[DIRCALL_RET:%[0-9]+]] = call i32 @func4()
+; ICALL-PROM-SAMPLEPGO: call i32 @func4(), !prof [[CALL_METADATA:![0-9]+]]
; ICALL-PROM: br label %if.end.icp
%call = call i32 %tmp(), !prof !1
; ICALL-PROM: if.false.orig_indirect:
@@ -54,3 +57,4 @@ entry:
; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1030, i32 570}
; ICALL-PROM: [[NEW_VP_METADATA]] = !{!"VP", i32 0, i64 570, i64 -4377547752858689819, i64 410}
+; ICALL-PROM-SAMPLEPGO: [[CALL_METADATA]] = !{!"branch_weights", i32 1030}
More information about the llvm-commits
mailing list