[llvm] r306231 - [PGO] Implementate profile counter regiser promotion
Xinliang David Li via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 24 17:26:43 PDT 2017
Author: davidxl
Date: Sat Jun 24 17:26:43 2017
New Revision: 306231
URL: http://llvm.org/viewvc/llvm-project?rev=306231&view=rev
Log:
[PGO] Implementate profile counter regiser promotion
Differential Revision: http://reviews.llvm.org/D34085
Added:
llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll
llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll
llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll
Modified:
llvm/trunk/include/llvm/Transforms/InstrProfiling.h
llvm/trunk/include/llvm/Transforms/Instrumentation.h
llvm/trunk/lib/Passes/PassBuilder.cpp
llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp
llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp
Modified: llvm/trunk/include/llvm/Transforms/InstrProfiling.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/InstrProfiling.h?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Transforms/InstrProfiling.h (original)
+++ llvm/trunk/include/llvm/Transforms/InstrProfiling.h Sat Jun 24 17:26:43 2017
@@ -28,6 +28,7 @@
namespace llvm {
class TargetLibraryInfo;
+using LoadStorePair = std::pair<Instruction *, Instruction *>;
/// Instrumentation based profiling lowering pass. This pass lowers
/// the profile instrumented code generated by FE or the IR based
@@ -60,11 +61,26 @@ private:
GlobalVariable *NamesVar;
size_t NamesSize;
+ // vector of counter load/store pairs to be register promoted.
+ std::vector<LoadStorePair> PromotionCandidates;
+
// The start value of precise value profile range for memory intrinsic sizes.
int64_t MemOPSizeRangeStart;
// The end value of precise value profile range for memory intrinsic sizes.
int64_t MemOPSizeRangeLast;
+ int64_t TotalCountersPromoted = 0;
+
+ /// Lower instrumentation intrinsics in the function. Returns true if there
+ /// any lowering.
+ bool lowerIntrinsics(Function *F);
+
+ /// Register-promote counter loads and stores in loops.
+ void promoteCounterLoadStores(Function *F);
+
+ /// Returns true if profile counter update register promotion is enabled.
+ bool isCounterPromotionEnabled() const;
+
/// Count the number of instrumented value sites for the function.
void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
Modified: llvm/trunk/include/llvm/Transforms/Instrumentation.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Instrumentation.h?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Transforms/Instrumentation.h (original)
+++ llvm/trunk/include/llvm/Transforms/Instrumentation.h Sat Jun 24 17:26:43 2017
@@ -116,6 +116,9 @@ struct InstrProfOptions {
// Add the 'noredzone' attribute to added runtime library calls.
bool NoRedZone = false;
+ // Do counter register promotion
+ bool DoCounterPromotion = false;
+
// Name of the profile file to use as output
std::string InstrProfileOutput;
Modified: llvm/trunk/lib/Passes/PassBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Passes/PassBuilder.cpp?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/lib/Passes/PassBuilder.cpp (original)
+++ llvm/trunk/lib/Passes/PassBuilder.cpp Sat Jun 24 17:26:43 2017
@@ -464,10 +464,15 @@ static void addPGOInstrPasses(ModulePass
if (RunProfileGen) {
MPM.addPass(PGOInstrumentationGen());
+ FunctionPassManager FPM;
+ FPM.addPass(createFunctionToLoopPassAdaptor(LoopRotatePass()));
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
// Add the profile lowering pass.
InstrProfOptions Options;
if (!ProfileGenFile.empty())
Options.InstrProfileOutput = ProfileGenFile;
+ Options.DoCounterPromotion = true;
MPM.addPass(InstrProfiling(Options));
}
Modified: llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp (original)
+++ llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp Sat Jun 24 17:26:43 2017
@@ -291,6 +291,8 @@ void PassManagerBuilder::addPGOInstrPass
InstrProfOptions Options;
if (!PGOInstrGen.empty())
Options.InstrProfileOutput = PGOInstrGen;
+ Options.DoCounterPromotion = true;
+ MPM.add(createLoopRotatePass());
MPM.add(createInstrProfilingLegacyPass(Options));
}
if (!PGOInstrUse.empty())
Modified: llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp (original)
+++ llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp Sat Jun 24 17:26:43 2017
@@ -19,12 +19,14 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
@@ -40,7 +42,10 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
+#include "llvm/Transforms/Utils/SSAUpdater.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -92,6 +97,35 @@ cl::opt<double> NumCountersPerValueSite(
// is usually smaller than 2.
cl::init(1.0));
+cl::opt<bool> AtomicCounterUpdatePromoted(
+ "atomic-counter-update-promoted", cl::ZeroOrMore,
+ cl::desc("Do counter update using atomic fetch add "
+ " for promoted counters only"),
+ cl::init(false));
+
+// If the option is not specified, the default behavior about whether
+// counter promotion is done depends on how instrumentaiton lowering
+// pipeline is setup, i.e., the default value of true of this option
+// does not mean the promotion will be done by default. Explicitly
+// setting this option can override the default behavior.
+cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
+ cl::desc("Do counter register promotion"),
+ cl::init(false));
+cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
+ cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(10),
+ cl::desc("Max number counter promotions per loop to avoid"
+ " increasing register pressure too much"));
+
+// A debug option
+cl::opt<int>
+ MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
+ cl::desc("Max number of allowed counter promotions"));
+
+cl::opt<bool> SpeculativeCounterPromotion(
+ cl::ZeroOrMore, "speculative-counter-promotion", cl::init(false),
+ cl::desc("Allow counter promotion for loops with multiple exiting blocks "
+ " or top-tested loops. "));
+
class InstrProfilingLegacyPass : public ModulePass {
InstrProfiling InstrProf;
@@ -116,6 +150,123 @@ public:
}
};
+/// A helper class to promote one counter RMW operation in the loop
+/// into register update.
+///
+/// RWM update for the counter will be sinked out of the loop after
+/// the transformation.
+///
+class PGOCounterPromoterHelper : public LoadAndStorePromoter {
+public:
+ PGOCounterPromoterHelper(Instruction *L, Instruction *S, SSAUpdater &SSA,
+ Value *Init, BasicBlock *PH,
+ ArrayRef<BasicBlock *> ExitBlocks,
+ ArrayRef<Instruction *> InsertPts)
+ : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
+ InsertPts(InsertPts) {
+ assert(isa<LoadInst>(L));
+ assert(isa<StoreInst>(S));
+ SSA.AddAvailableValue(PH, Init);
+ }
+ void doExtraRewritesBeforeFinalDeletion() const override {
+ for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
+ BasicBlock *ExitBlock = ExitBlocks[i];
+ Instruction *InsertPos = InsertPts[i];
+ // Get LiveIn value into the ExitBlock. If there are multiple
+ // predecessors, the value is defined by a PHI node in this
+ // block.
+ Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
+ Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
+ IRBuilder<> Builder(InsertPos);
+ if (AtomicCounterUpdatePromoted)
+ Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
+ AtomicOrdering::SequentiallyConsistent);
+ else {
+ LoadInst *OldVal = Builder.CreateLoad(Addr, "pgocount.promoted");
+ auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
+ Builder.CreateStore(NewVal, Addr);
+ }
+ }
+ }
+
+private:
+ Instruction *Store;
+ ArrayRef<BasicBlock *> ExitBlocks;
+ ArrayRef<Instruction *> InsertPts;
+};
+
+/// A helper class to do register promotion for all profile counter
+/// updates in a loop.
+///
+class PGOCounterPromoter {
+public:
+ PGOCounterPromoter(ArrayRef<LoadStorePair> Cands, Loop &Loop)
+ : Candidates(Cands), ExitBlocks(), InsertPts(), ParentLoop(Loop) {
+
+ SmallVector<BasicBlock *, 8> LoopExitBlocks;
+ SmallPtrSet<BasicBlock *, 8> BlockSet;
+ ParentLoop.getExitBlocks(LoopExitBlocks);
+
+ for (BasicBlock *ExitBlock : LoopExitBlocks) {
+ if (BlockSet.insert(ExitBlock).second) {
+ ExitBlocks.push_back(ExitBlock);
+ InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
+ }
+ }
+ }
+
+ bool run(int64_t *NumPromoted) {
+ // We can't insert into a catchswitch.
+ bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) {
+ return isa<CatchSwitchInst>(Exit->getTerminator());
+ });
+
+ if (HasCatchSwitch)
+ return false;
+
+ if (!ParentLoop.hasDedicatedExits())
+ return false;
+
+ BasicBlock *PH = ParentLoop.getLoopPreheader();
+ if (!PH)
+ return false;
+
+ BasicBlock *H = ParentLoop.getHeader();
+ bool TopTested =
+ ((ParentLoop.getBlocks().size() > 1) && ParentLoop.isLoopExiting(H));
+ if (!SpeculativeCounterPromotion &&
+ (TopTested || ParentLoop.getExitingBlock() == nullptr))
+ return false;
+
+ unsigned Promoted = 0;
+ for (auto &Cand : Candidates) {
+
+ SmallVector<PHINode *, 4> NewPHIs;
+ SSAUpdater SSA(&NewPHIs);
+ Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
+ PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
+ PH, ExitBlocks, InsertPts);
+ Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
+ Promoted++;
+ if (Promoted >= MaxNumOfPromotionsPerLoop)
+ break;
+ (*NumPromoted)++;
+ if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
+ break;
+ }
+
+ DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
+ << ParentLoop.getLoopDepth() << ")\n");
+ return Promoted != 0;
+ }
+
+private:
+ ArrayRef<LoadStorePair> Candidates;
+ SmallVector<BasicBlock *, 8> ExitBlocks;
+ SmallVector<Instruction *, 8> InsertPts;
+ Loop &ParentLoop;
+};
+
} // end anonymous namespace
PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
@@ -147,6 +298,63 @@ static InstrProfIncrementInst *castToInc
return dyn_cast<InstrProfIncrementInst>(Instr);
}
+bool InstrProfiling::lowerIntrinsics(Function *F) {
+ bool MadeChange = false;
+ PromotionCandidates.clear();
+ for (BasicBlock &BB : *F) {
+ for (auto I = BB.begin(), E = BB.end(); I != E;) {
+ auto Instr = I++;
+ InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
+ if (Inc) {
+ lowerIncrement(Inc);
+ MadeChange = true;
+ } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
+ lowerValueProfileInst(Ind);
+ MadeChange = true;
+ }
+ }
+ }
+
+ if (!MadeChange)
+ return false;
+
+ promoteCounterLoadStores(F);
+ return true;
+}
+
+bool InstrProfiling::isCounterPromotionEnabled() const {
+ if (DoCounterPromotion.getNumOccurrences() > 0)
+ return DoCounterPromotion;
+
+ return Options.DoCounterPromotion;
+}
+
+void InstrProfiling::promoteCounterLoadStores(Function *F) {
+ if (!isCounterPromotionEnabled())
+ return;
+
+ DominatorTree DT(*F);
+ LoopInfo LI(DT);
+ DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
+
+ for (const auto &LoadStore : PromotionCandidates) {
+ auto *CounterLoad = LoadStore.first;
+ auto *CounterStore = LoadStore.second;
+ BasicBlock *BB = CounterLoad->getParent();
+ Loop *ParentLoop = LI.getLoopFor(BB);
+ if (!ParentLoop)
+ continue;
+ LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
+ }
+
+ SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
+
+ for (auto *Loop : Loops) {
+ PGOCounterPromoter Promoter(LoopPromotionCandidates[Loop], *Loop);
+ Promoter.run(&TotalCountersPromoted);
+ }
+}
+
bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
bool MadeChange = false;
@@ -179,18 +387,7 @@ bool InstrProfiling::run(Module &M, cons
}
for (Function &F : M)
- for (BasicBlock &BB : F)
- for (auto I = BB.begin(), E = BB.end(); I != E;) {
- auto Instr = I++;
- InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
- if (Inc) {
- lowerIncrement(Inc);
- MadeChange = true;
- } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
- lowerValueProfileInst(Ind);
- MadeChange = true;
- }
- }
+ MadeChange |= lowerIntrinsics(&F);
if (GlobalVariable *CoverageNamesVar =
M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
@@ -303,9 +500,12 @@ void InstrProfiling::lowerIncrement(Inst
IRBuilder<> Builder(Inc);
uint64_t Index = Inc->getIndex()->getZExtValue();
Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
- Value *Count = Builder.CreateLoad(Addr, "pgocount");
- Count = Builder.CreateAdd(Count, Inc->getStep());
- Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
+ Value *Load = Builder.CreateLoad(Addr, "pgocount");
+ auto *Count = Builder.CreateAdd(Load, Inc->getStep());
+ auto *Store = Builder.CreateStore(Count, Addr);
+ Inc->replaceAllUsesWith(Store);
+ if (isCounterPromotionEnabled())
+ PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
Inc->eraseFromParent();
}
Added: llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll?rev=306231&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll (added)
+++ llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll Sat Jun 24 17:26:43 2017
@@ -0,0 +1,68 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -atomic-counter-update-promoted -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -atomic-counter-update-promoted -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s
+
+define void @foo(i32 %n, i32 %N) {
+; PROMO-LABEL: @foo
+bb:
+ %tmp = add nsw i32 %n, 1
+ %tmp1 = add nsw i32 %n, -1
+ br label %bb2
+
+bb2: ; preds = %bb9, %bb
+; PROMO: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+ %i.0 = phi i32 [ 0, %bb ], [ %tmp10, %bb9 ]
+ %tmp3 = icmp slt i32 %i.0, %tmp
+ br i1 %tmp3, label %bb4, label %bb5
+
+bb4: ; preds = %bb2
+ tail call void @bar(i32 1)
+ br label %bb9
+
+bb5: ; preds = %bb2
+ %tmp6 = icmp slt i32 %i.0, %tmp1
+ br i1 %tmp6, label %bb7, label %bb8
+
+bb7: ; preds = %bb5
+ tail call void @bar(i32 2)
+ br label %bb9
+
+bb8: ; preds = %bb5
+ tail call void @bar(i32 3)
+ br label %bb9
+
+bb9: ; preds = %bb8, %bb7, %bb4
+; PROMO: %[[LIVEOUT3:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[LIVEOUT2:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[LIVEOUT1:[a-z0-9]+]] = phi {{.*}}
+ %tmp10 = add nsw i32 %i.0, 1
+ %tmp11 = icmp slt i32 %tmp10, %N
+ br i1 %tmp11, label %bb2, label %bb12
+
+bb12: ; preds = %bb9
+ ret void
+; NONATOMIC_PROMO: %[[PROMO1:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 0)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO1]], %[[LIVEOUT1]]
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; NONATOMIC_PROMO-NEXT: %[[PROMO2:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 1)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO2]], %[[LIVEOUT2]]
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; NONATOMIC_PROMO-NEXT: %[[PROMO3:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 2)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO3]], %[[LIVEOUT3]]
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; ATOMIC_PROMO: atomicrmw add {{.*}} @__profc_foo{{.*}}0), i64 %[[LIVEOUT1]] seq_cst
+; ATOMIC_PROMO-NEXT: atomicrmw add {{.*}} @__profc_foo{{.*}}1), i64 %[[LIVEOUT2]] seq_cst
+; ATOMIC_PROMO-NEXT: atomicrmw add {{.*}} @__profc_foo{{.*}}2), i64 %[[LIVEOUT3]] seq_cst
+; PROMO: {{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
+; PROMO-NOT: @__profc_foo
+
+
+}
+
+declare void @bar(i32)
Added: llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll?rev=306231&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll (added)
+++ llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll Sat Jun 24 17:26:43 2017
@@ -0,0 +1,74 @@
+; RUN: opt < %s -instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s
+; RUN: opt < %s --passes=instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s
+
+$__llvm_profile_raw_version = comdat any
+
+ at g = common local_unnamed_addr global i32 0, align 4
+ at __llvm_profile_raw_version = constant i64 72057594037927940, comdat
+ at __profn_foo = private constant [3 x i8] c"foo"
+
+define void @foo(i32 %arg) local_unnamed_addr {
+bb:
+ %tmp = add nsw i32 %arg, -1
+ br label %bb1
+
+bb1: ; preds = %bb11, %bb
+ %tmp2 = phi i32 [ 0, %bb ], [ %tmp12, %bb11 ]
+ %tmp3 = icmp sgt i32 %tmp2, %arg
+ br i1 %tmp3, label %bb7, label %bb4
+
+bb4: ; preds = %bb1
+ call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 1)
+ tail call void @bar(i32 1)
+ %tmp5 = load i32, i32* @g, align 4
+ %tmp6 = icmp sgt i32 %tmp5, 100
+ br i1 %tmp6, label %bb14, label %bb11
+
+bb7: ; preds = %bb1
+ %tmp8 = icmp slt i32 %tmp2, %tmp
+ br i1 %tmp8, label %bb9, label %bb10
+
+bb9: ; preds = %bb7
+ call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 2)
+ tail call void @bar(i32 2)
+ br label %bb11
+
+bb10: ; preds = %bb7
+ call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 3)
+ tail call void @bar(i32 3)
+ br label %bb11
+
+bb11: ; preds = %bb10, %bb9, %bb4
+ %tmp12 = add nuw nsw i32 %tmp2, 1
+ %tmp13 = icmp slt i32 %tmp2, 99
+ br i1 %tmp13, label %bb1, label %bb14
+
+bb14: ; preds = %bb4.bb14_crit_edge, %bb11
+ tail call void @bar(i32 0)
+ br label %bb15
+; PROMO-LABEL: bb14:
+; PROMO: %[[MERGE1:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[MERGE2:[a-z0-9.]+]] = phi {{.*}}
+; PROMO-NEXT: %[[MERGE3:[a-z0-9.]+]] = phi {{.*}}
+; PROMO-NEXT: %[[PROMO3:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO3]], %[[MERGE3]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: %[[PROMO2:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO2]], %[[MERGE2]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT: %[[PROMO1:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}3)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO1]], %[[MERGE1]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}3)
+
+bb15: ; preds = %bb14
+ call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 4)
+ tail call void @bar(i32 1)
+ ret void
+}
+
+declare void @bar(i32) local_unnamed_addr
+
+; Function Attrs: nounwind
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #0
+
+attributes #0 = { nounwind }
Added: llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll?rev=306231&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll (added)
+++ llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll Sat Jun 24 17:26:43 2017
@@ -0,0 +1,80 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s
+
+ at g = common local_unnamed_addr global i32 0, align 4
+
+define void @foo(i32 %arg) local_unnamed_addr {
+; PROMO-LABEL: @foo
+bb:
+ %tmp = add nsw i32 %arg, -1
+ br label %bb1
+bb1: ; preds = %bb11, %bb
+ %tmp2 = phi i32 [ 0, %bb ], [ %tmp12, %bb11 ]
+ %tmp3 = icmp sgt i32 %tmp2, %arg
+ br i1 %tmp3, label %bb7, label %bb4
+
+bb4: ; preds = %bb1
+ tail call void @bar(i32 1)
+ %tmp5 = load i32, i32* @g, align 4
+ %tmp6 = icmp sgt i32 %tmp5, 100
+ br i1 %tmp6, label %bb15_0, label %bb11
+
+bb7: ; preds = %bb1
+ %tmp8 = icmp slt i32 %tmp2, %tmp
+ br i1 %tmp8, label %bb9, label %bb10
+
+bb9: ; preds = %bb7
+ tail call void @bar(i32 2)
+ br label %bb11
+
+bb10: ; preds = %bb7
+ tail call void @bar(i32 3)
+ br label %bb11
+
+bb11: ; preds = %bb10, %bb9, %bb4
+ %tmp12 = add nuw nsw i32 %tmp2, 1
+ %tmp13 = icmp slt i32 %tmp2, 99
+ br i1 %tmp13, label %bb1, label %bb14
+
+bb14: ; preds = %bb11
+; PROMO-LABEL: bb14:
+ tail call void @bar(i32 0)
+ br label %bb15
+; PROMO: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 0)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT: %pgocount{{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
+
+
+bb15_0: ; preds = %bb11
+; PROMO-LABEL: bb15_0:
+ br label %bb15
+; PROMO: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 0)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT: %pgocount{{.*}} = load {{.*}} @__profc_foo{{.*}} 4)
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}4)
+; PROMO-NOT: @__profc_foo
+
+
+bb15: ; preds = %bb14, %bb4
+ tail call void @bar(i32 1)
+ ret void
+}
+
+declare void @bar(i32) local_unnamed_addr
More information about the llvm-commits
mailing list