[llvm] r306231 - [PGO] Implementate profile counter regiser promotion

Xinliang David Li via llvm-commits llvm-commits at lists.llvm.org
Sat Jun 24 17:26:43 PDT 2017


Author: davidxl
Date: Sat Jun 24 17:26:43 2017
New Revision: 306231

URL: http://llvm.org/viewvc/llvm-project?rev=306231&view=rev
Log:
[PGO] Implementate profile counter regiser promotion

Differential Revision: http://reviews.llvm.org/D34085

Added:
    llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll
    llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll
    llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll
Modified:
    llvm/trunk/include/llvm/Transforms/InstrProfiling.h
    llvm/trunk/include/llvm/Transforms/Instrumentation.h
    llvm/trunk/lib/Passes/PassBuilder.cpp
    llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp
    llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp

Modified: llvm/trunk/include/llvm/Transforms/InstrProfiling.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/InstrProfiling.h?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Transforms/InstrProfiling.h (original)
+++ llvm/trunk/include/llvm/Transforms/InstrProfiling.h Sat Jun 24 17:26:43 2017
@@ -28,6 +28,7 @@
 namespace llvm {
 
 class TargetLibraryInfo;
+using LoadStorePair = std::pair<Instruction *, Instruction *>;
 
 /// Instrumentation based profiling lowering pass. This pass lowers
 /// the profile instrumented code generated by FE or the IR based
@@ -60,11 +61,26 @@ private:
   GlobalVariable *NamesVar;
   size_t NamesSize;
 
+  // vector of counter load/store pairs to be register promoted.
+  std::vector<LoadStorePair> PromotionCandidates;
+
   // The start value of precise value profile range for memory intrinsic sizes.
   int64_t MemOPSizeRangeStart;
   // The end value of precise value profile range for memory intrinsic sizes.
   int64_t MemOPSizeRangeLast;
 
+  int64_t TotalCountersPromoted = 0;
+
+  /// Lower instrumentation intrinsics in the function. Returns true if there
+  /// any lowering.
+  bool lowerIntrinsics(Function *F);
+
+  /// Register-promote counter loads and stores in loops.
+  void promoteCounterLoadStores(Function *F);
+
+  /// Returns true if profile counter update register promotion is enabled.
+  bool isCounterPromotionEnabled() const;
+
   /// Count the number of instrumented value sites for the function.
   void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
 

Modified: llvm/trunk/include/llvm/Transforms/Instrumentation.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Instrumentation.h?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Transforms/Instrumentation.h (original)
+++ llvm/trunk/include/llvm/Transforms/Instrumentation.h Sat Jun 24 17:26:43 2017
@@ -116,6 +116,9 @@ struct InstrProfOptions {
   // Add the 'noredzone' attribute to added runtime library calls.
   bool NoRedZone = false;
 
+  // Do counter register promotion
+  bool DoCounterPromotion = false;
+
   // Name of the profile file to use as output
   std::string InstrProfileOutput;
 

Modified: llvm/trunk/lib/Passes/PassBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Passes/PassBuilder.cpp?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/lib/Passes/PassBuilder.cpp (original)
+++ llvm/trunk/lib/Passes/PassBuilder.cpp Sat Jun 24 17:26:43 2017
@@ -464,10 +464,15 @@ static void addPGOInstrPasses(ModulePass
   if (RunProfileGen) {
     MPM.addPass(PGOInstrumentationGen());
 
+    FunctionPassManager FPM;
+    FPM.addPass(createFunctionToLoopPassAdaptor(LoopRotatePass()));
+    MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
     // Add the profile lowering pass.
     InstrProfOptions Options;
     if (!ProfileGenFile.empty())
       Options.InstrProfileOutput = ProfileGenFile;
+    Options.DoCounterPromotion = true;
     MPM.addPass(InstrProfiling(Options));
   }
 

Modified: llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp (original)
+++ llvm/trunk/lib/Transforms/IPO/PassManagerBuilder.cpp Sat Jun 24 17:26:43 2017
@@ -291,6 +291,8 @@ void PassManagerBuilder::addPGOInstrPass
     InstrProfOptions Options;
     if (!PGOInstrGen.empty())
       Options.InstrProfileOutput = PGOInstrGen;
+    Options.DoCounterPromotion = true;
+    MPM.add(createLoopRotatePass());
     MPM.add(createInstrProfilingLegacyPass(Options));
   }
   if (!PGOInstrUse.empty())

Modified: llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp?rev=306231&r1=306230&r2=306231&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp (original)
+++ llvm/trunk/lib/Transforms/Instrumentation/InstrProfiling.cpp Sat Jun 24 17:26:43 2017
@@ -19,12 +19,14 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -40,7 +42,10 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
+#include "llvm/Transforms/Utils/SSAUpdater.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
@@ -92,6 +97,35 @@ cl::opt<double> NumCountersPerValueSite(
     // is usually smaller than 2.
     cl::init(1.0));
 
+cl::opt<bool> AtomicCounterUpdatePromoted(
+    "atomic-counter-update-promoted", cl::ZeroOrMore,
+    cl::desc("Do counter update using atomic fetch add "
+             " for promoted counters only"),
+    cl::init(false));
+
+// If the option is not specified, the default behavior about whether
+// counter promotion is done depends on how instrumentaiton lowering
+// pipeline is setup, i.e., the default value of true of this option
+// does not mean the promotion will be done by default. Explicitly
+// setting this option can override the default behavior.
+cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
+                                 cl::desc("Do counter register promotion"),
+                                 cl::init(false));
+cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
+    cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(10),
+    cl::desc("Max number counter promotions per loop to avoid"
+             " increasing register pressure too much"));
+
+// A debug option
+cl::opt<int>
+    MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
+                       cl::desc("Max number of allowed counter promotions"));
+
+cl::opt<bool> SpeculativeCounterPromotion(
+    cl::ZeroOrMore, "speculative-counter-promotion", cl::init(false),
+    cl::desc("Allow counter promotion for loops with multiple exiting blocks "
+             " or top-tested loops. "));
+
 class InstrProfilingLegacyPass : public ModulePass {
   InstrProfiling InstrProf;
 
@@ -116,6 +150,123 @@ public:
   }
 };
 
+/// A helper class to promote one counter RMW operation in the loop
+/// into register update.
+///
+/// RWM update for the counter will be sinked out of the loop after
+/// the transformation.
+///
+class PGOCounterPromoterHelper : public LoadAndStorePromoter {
+public:
+  PGOCounterPromoterHelper(Instruction *L, Instruction *S, SSAUpdater &SSA,
+                           Value *Init, BasicBlock *PH,
+                           ArrayRef<BasicBlock *> ExitBlocks,
+                           ArrayRef<Instruction *> InsertPts)
+      : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
+        InsertPts(InsertPts) {
+    assert(isa<LoadInst>(L));
+    assert(isa<StoreInst>(S));
+    SSA.AddAvailableValue(PH, Init);
+  }
+  void doExtraRewritesBeforeFinalDeletion() const override {
+    for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
+      BasicBlock *ExitBlock = ExitBlocks[i];
+      Instruction *InsertPos = InsertPts[i];
+      // Get LiveIn value into the ExitBlock. If there are multiple
+      // predecessors, the value is defined by a PHI node in this
+      // block.
+      Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
+      Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
+      IRBuilder<> Builder(InsertPos);
+      if (AtomicCounterUpdatePromoted)
+        Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
+                                AtomicOrdering::SequentiallyConsistent);
+      else {
+        LoadInst *OldVal = Builder.CreateLoad(Addr, "pgocount.promoted");
+        auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
+        Builder.CreateStore(NewVal, Addr);
+      }
+    }
+  }
+
+private:
+  Instruction *Store;
+  ArrayRef<BasicBlock *> ExitBlocks;
+  ArrayRef<Instruction *> InsertPts;
+};
+
+/// A helper class to do register promotion for all profile counter
+/// updates in a loop.
+///
+class PGOCounterPromoter {
+public:
+  PGOCounterPromoter(ArrayRef<LoadStorePair> Cands, Loop &Loop)
+      : Candidates(Cands), ExitBlocks(), InsertPts(), ParentLoop(Loop) {
+
+    SmallVector<BasicBlock *, 8> LoopExitBlocks;
+    SmallPtrSet<BasicBlock *, 8> BlockSet;
+    ParentLoop.getExitBlocks(LoopExitBlocks);
+
+    for (BasicBlock *ExitBlock : LoopExitBlocks) {
+      if (BlockSet.insert(ExitBlock).second) {
+        ExitBlocks.push_back(ExitBlock);
+        InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
+      }
+    }
+  }
+
+  bool run(int64_t *NumPromoted) {
+    // We can't insert into a catchswitch.
+    bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) {
+      return isa<CatchSwitchInst>(Exit->getTerminator());
+    });
+
+    if (HasCatchSwitch)
+      return false;
+
+    if (!ParentLoop.hasDedicatedExits())
+      return false;
+
+    BasicBlock *PH = ParentLoop.getLoopPreheader();
+    if (!PH)
+      return false;
+
+    BasicBlock *H = ParentLoop.getHeader();
+    bool TopTested =
+        ((ParentLoop.getBlocks().size() > 1) && ParentLoop.isLoopExiting(H));
+    if (!SpeculativeCounterPromotion &&
+        (TopTested || ParentLoop.getExitingBlock() == nullptr))
+      return false;
+
+    unsigned Promoted = 0;
+    for (auto &Cand : Candidates) {
+
+      SmallVector<PHINode *, 4> NewPHIs;
+      SSAUpdater SSA(&NewPHIs);
+      Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
+      PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
+                                        PH, ExitBlocks, InsertPts);
+      Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
+      Promoted++;
+      if (Promoted >= MaxNumOfPromotionsPerLoop)
+        break;
+      (*NumPromoted)++;
+      if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
+        break;
+    }
+
+    DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
+                 << ParentLoop.getLoopDepth() << ")\n");
+    return Promoted != 0;
+  }
+
+private:
+  ArrayRef<LoadStorePair> Candidates;
+  SmallVector<BasicBlock *, 8> ExitBlocks;
+  SmallVector<Instruction *, 8> InsertPts;
+  Loop &ParentLoop;
+};
+
 } // end anonymous namespace
 
 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
@@ -147,6 +298,63 @@ static InstrProfIncrementInst *castToInc
   return dyn_cast<InstrProfIncrementInst>(Instr);
 }
 
+bool InstrProfiling::lowerIntrinsics(Function *F) {
+  bool MadeChange = false;
+  PromotionCandidates.clear();
+  for (BasicBlock &BB : *F) {
+    for (auto I = BB.begin(), E = BB.end(); I != E;) {
+      auto Instr = I++;
+      InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
+      if (Inc) {
+        lowerIncrement(Inc);
+        MadeChange = true;
+      } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
+        lowerValueProfileInst(Ind);
+        MadeChange = true;
+      }
+    }
+  }
+
+  if (!MadeChange)
+    return false;
+
+  promoteCounterLoadStores(F);
+  return true;
+}
+
+bool InstrProfiling::isCounterPromotionEnabled() const {
+  if (DoCounterPromotion.getNumOccurrences() > 0)
+    return DoCounterPromotion;
+
+  return Options.DoCounterPromotion;
+}
+
+void InstrProfiling::promoteCounterLoadStores(Function *F) {
+  if (!isCounterPromotionEnabled())
+    return;
+
+  DominatorTree DT(*F);
+  LoopInfo LI(DT);
+  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
+
+  for (const auto &LoadStore : PromotionCandidates) {
+    auto *CounterLoad = LoadStore.first;
+    auto *CounterStore = LoadStore.second;
+    BasicBlock *BB = CounterLoad->getParent();
+    Loop *ParentLoop = LI.getLoopFor(BB);
+    if (!ParentLoop)
+      continue;
+    LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
+  }
+
+  SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
+
+  for (auto *Loop : Loops) {
+    PGOCounterPromoter Promoter(LoopPromotionCandidates[Loop], *Loop);
+    Promoter.run(&TotalCountersPromoted);
+  }
+}
+
 bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
   bool MadeChange = false;
 
@@ -179,18 +387,7 @@ bool InstrProfiling::run(Module &M, cons
   }
 
   for (Function &F : M)
-    for (BasicBlock &BB : F)
-      for (auto I = BB.begin(), E = BB.end(); I != E;) {
-        auto Instr = I++;
-        InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
-        if (Inc) {
-          lowerIncrement(Inc);
-          MadeChange = true;
-        } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
-          lowerValueProfileInst(Ind);
-          MadeChange = true;
-        }
-      }
+    MadeChange |= lowerIntrinsics(&F);
 
   if (GlobalVariable *CoverageNamesVar =
           M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
@@ -303,9 +500,12 @@ void InstrProfiling::lowerIncrement(Inst
   IRBuilder<> Builder(Inc);
   uint64_t Index = Inc->getIndex()->getZExtValue();
   Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
-  Value *Count = Builder.CreateLoad(Addr, "pgocount");
-  Count = Builder.CreateAdd(Count, Inc->getStep());
-  Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
+  Value *Load = Builder.CreateLoad(Addr, "pgocount");
+  auto *Count = Builder.CreateAdd(Load, Inc->getStep());
+  auto *Store = Builder.CreateStore(Count, Addr);
+  Inc->replaceAllUsesWith(Store);
+  if (isCounterPromotionEnabled())
+    PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
   Inc->eraseFromParent();
 }
 

Added: llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll?rev=306231&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll (added)
+++ llvm/trunk/test/Transforms/PGOProfile/counter_promo.ll Sat Jun 24 17:26:43 2017
@@ -0,0 +1,68 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s 
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -atomic-counter-update-promoted -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -atomic-counter-update-promoted -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s 
+
+define void @foo(i32 %n, i32 %N) {
+; PROMO-LABEL: @foo
+bb:
+  %tmp = add nsw i32 %n, 1
+  %tmp1 = add nsw i32 %n, -1
+  br label %bb2
+
+bb2:                                              ; preds = %bb9, %bb
+; PROMO: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+  %i.0 = phi i32 [ 0, %bb ], [ %tmp10, %bb9 ]
+  %tmp3 = icmp slt i32 %i.0, %tmp
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb2
+  tail call void @bar(i32 1)
+  br label %bb9
+
+bb5:                                              ; preds = %bb2
+  %tmp6 = icmp slt i32 %i.0, %tmp1
+  br i1 %tmp6, label %bb7, label %bb8
+
+bb7:                                              ; preds = %bb5
+  tail call void @bar(i32 2)
+  br label %bb9
+
+bb8:                                              ; preds = %bb5
+  tail call void @bar(i32 3)
+  br label %bb9
+
+bb9:                                              ; preds = %bb8, %bb7, %bb4
+; PROMO: %[[LIVEOUT3:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[LIVEOUT2:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[LIVEOUT1:[a-z0-9]+]] = phi {{.*}}
+  %tmp10 = add nsw i32 %i.0, 1
+  %tmp11 = icmp slt i32 %tmp10, %N
+  br i1 %tmp11, label %bb2, label %bb12
+
+bb12:                                             ; preds = %bb9
+  ret void
+; NONATOMIC_PROMO: %[[PROMO1:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 0)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO1]], %[[LIVEOUT1]] 
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; NONATOMIC_PROMO-NEXT: %[[PROMO2:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 1)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO2]], %[[LIVEOUT2]]
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; NONATOMIC_PROMO-NEXT: %[[PROMO3:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 2)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO3]], %[[LIVEOUT3]]
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; ATOMIC_PROMO: atomicrmw add {{.*}} @__profc_foo{{.*}}0), i64 %[[LIVEOUT1]] seq_cst
+; ATOMIC_PROMO-NEXT: atomicrmw add {{.*}} @__profc_foo{{.*}}1), i64 %[[LIVEOUT2]] seq_cst
+; ATOMIC_PROMO-NEXT: atomicrmw add {{.*}} @__profc_foo{{.*}}2), i64 %[[LIVEOUT3]] seq_cst
+; PROMO: {{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
+; PROMO-NOT: @__profc_foo
+
+
+}
+
+declare void @bar(i32)

Added: llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll?rev=306231&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll (added)
+++ llvm/trunk/test/Transforms/PGOProfile/counter_promo_exit_merge.ll Sat Jun 24 17:26:43 2017
@@ -0,0 +1,74 @@
+; RUN: opt < %s -instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s
+; RUN: opt < %s --passes=instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s
+
+$__llvm_profile_raw_version = comdat any
+
+ at g = common local_unnamed_addr global i32 0, align 4
+ at __llvm_profile_raw_version = constant i64 72057594037927940, comdat
+ at __profn_foo = private constant [3 x i8] c"foo"
+
+define void @foo(i32 %arg) local_unnamed_addr {
+bb:
+  %tmp = add nsw i32 %arg, -1
+  br label %bb1
+
+bb1:                                              ; preds = %bb11, %bb
+  %tmp2 = phi i32 [ 0, %bb ], [ %tmp12, %bb11 ]
+  %tmp3 = icmp sgt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb7, label %bb4
+
+bb4:                                              ; preds = %bb1
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 1)
+  tail call void @bar(i32 1)
+  %tmp5 = load i32, i32* @g, align 4
+  %tmp6 = icmp sgt i32 %tmp5, 100
+  br i1 %tmp6, label %bb14, label %bb11
+
+bb7:                                              ; preds = %bb1
+  %tmp8 = icmp slt i32 %tmp2, %tmp
+  br i1 %tmp8, label %bb9, label %bb10
+
+bb9:                                              ; preds = %bb7
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 2)
+  tail call void @bar(i32 2)
+  br label %bb11
+
+bb10:                                             ; preds = %bb7
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 3)
+  tail call void @bar(i32 3)
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb9, %bb4
+  %tmp12 = add nuw nsw i32 %tmp2, 1
+  %tmp13 = icmp slt i32 %tmp2, 99
+  br i1 %tmp13, label %bb1, label %bb14
+
+bb14:                                             ; preds = %bb4.bb14_crit_edge, %bb11
+  tail call void @bar(i32 0)
+  br label %bb15
+; PROMO-LABEL: bb14:
+; PROMO: %[[MERGE1:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[MERGE2:[a-z0-9.]+]] = phi {{.*}}
+; PROMO-NEXT: %[[MERGE3:[a-z0-9.]+]] = phi {{.*}}
+; PROMO-NEXT: %[[PROMO3:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO3]], %[[MERGE3]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: %[[PROMO2:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO2]], %[[MERGE2]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT: %[[PROMO1:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}3)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO1]], %[[MERGE1]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}3)
+
+bb15:                                             ; preds = %bb14
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 4)
+  tail call void @bar(i32 1)
+  ret void
+}
+
+declare void @bar(i32) local_unnamed_addr
+
+; Function Attrs: nounwind
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #0
+
+attributes #0 = { nounwind }

Added: llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll?rev=306231&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll (added)
+++ llvm/trunk/test/Transforms/PGOProfile/counter_promo_mexits.ll Sat Jun 24 17:26:43 2017
@@ -0,0 +1,80 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s
+
+ at g = common local_unnamed_addr global i32 0, align 4
+
+define void @foo(i32 %arg) local_unnamed_addr {
+; PROMO-LABEL: @foo
+bb:
+  %tmp = add nsw i32 %arg, -1
+  br label %bb1
+bb1:                                              ; preds = %bb11, %bb
+  %tmp2 = phi i32 [ 0, %bb ], [ %tmp12, %bb11 ]
+  %tmp3 = icmp sgt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb7, label %bb4
+
+bb4:                                              ; preds = %bb1
+  tail call void @bar(i32 1)
+  %tmp5 = load i32, i32* @g, align 4
+  %tmp6 = icmp sgt i32 %tmp5, 100
+  br i1 %tmp6, label %bb15_0, label %bb11
+
+bb7:                                              ; preds = %bb1
+  %tmp8 = icmp slt i32 %tmp2, %tmp
+  br i1 %tmp8, label %bb9, label %bb10
+
+bb9:                                              ; preds = %bb7
+  tail call void @bar(i32 2)
+  br label %bb11
+
+bb10:                                             ; preds = %bb7
+  tail call void @bar(i32 3)
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb9, %bb4
+  %tmp12 = add nuw nsw i32 %tmp2, 1
+  %tmp13 = icmp slt i32 %tmp2, 99
+  br i1 %tmp13, label %bb1, label %bb14
+
+bb14:                                             ; preds = %bb11
+; PROMO-LABEL: bb14:
+  tail call void @bar(i32 0)
+  br label %bb15
+; PROMO:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 0)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; PROMO-NEXT:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT:  %pgocount{{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
+
+
+bb15_0:                                             ; preds = %bb11
+; PROMO-LABEL: bb15_0:
+  br label %bb15
+; PROMO:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 0)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; PROMO-NEXT:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT:  %pgocount{{.*}} = load {{.*}} @__profc_foo{{.*}} 4)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}4)
+; PROMO-NOT: @__profc_foo
+
+
+bb15:                                             ; preds = %bb14, %bb4
+  tail call void @bar(i32 1)
+  ret void
+}
+
+declare void @bar(i32) local_unnamed_addr




More information about the llvm-commits mailing list