[llvm] 7397905 - [SampleFDO] Third Try: Refactor SampleProfile.cpp
Rong Xu via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 17 15:33:28 PST 2021
Author: Rong Xu
Date: 2021-02-17T15:31:50-08:00
New Revision: 7397905ab0a081afe14f2a342082accc0bbb8695
URL: https://github.com/llvm/llvm-project/commit/7397905ab0a081afe14f2a342082accc0bbb8695
DIFF: https://github.com/llvm/llvm-project/commit/7397905ab0a081afe14f2a342082accc0bbb8695.diff
LOG: [SampleFDO] Third Try: Refactor SampleProfile.cpp
Apply the patch for the third time after fixing buildbot failures.
Refactor SampleProfile.cpp to use the core code in CodeGen.
The main changes are:
(1) Move SampleProfileLoaderBaseImpl class to a header file.
(2) Split SampleCoverageTracker to a head file and a cpp file.
(3) Move the common codes (common options and callsiteIsHot())
to the common cpp file.
(4) Add inline keyword to avoid duplicated symbols -- they will
be removed later when the class is changed to a template.
Differential Revision: https://reviews.llvm.org/D96455
Added:
llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h
llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp
Modified:
llvm/lib/Transforms/IPO/SampleProfile.cpp
llvm/lib/Transforms/Utils/CMakeLists.txt
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
new file mode 100644
index 000000000000..0d53780ebffd
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
@@ -0,0 +1,859 @@
+////===- SampleProfileLoadBaseImpl.h - Profile loader base impl --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file provides the interface for the sampled PGO profile loader base
+/// implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_SAMPLEPROFILELOADERIMPL_H
+#define LLVM_TRANSFORMS_UTILS_SAMPLEPROFILELOADERIMPL_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/DebugLoc.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/ProfileData/SampleProf.h"
+#include "llvm/ProfileData/SampleProfReader.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/GenericDomTree.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
+
+namespace llvm {
+using namespace llvm;
+using namespace sampleprof;
+using namespace sampleprofutil;
+using ProfileCount = Function::ProfileCount;
+
+#define DEBUG_TYPE "sample-profile-impl"
+
+using BlockWeightMap = DenseMap<const BasicBlock *, uint64_t>;
+using EquivalenceClassMap = DenseMap<const BasicBlock *, const BasicBlock *>;
+using Edge = std::pair<const BasicBlock *, const BasicBlock *>;
+using EdgeWeightMap = DenseMap<Edge, uint64_t>;
+using BlockEdgeMap =
+ DenseMap<const BasicBlock *, SmallVector<const BasicBlock *, 8>>;
+
+extern cl::opt<unsigned> SampleProfileMaxPropagateIterations;
+extern cl::opt<unsigned> SampleProfileRecordCoverage;
+extern cl::opt<unsigned> SampleProfileSampleCoverage;
+extern cl::opt<bool> NoWarnSampleUnused;
+
+class SampleProfileLoaderBaseImpl {
+public:
+ SampleProfileLoaderBaseImpl(std::string Name) : Filename(Name) {}
+ void dump() { Reader->dump(); }
+
+protected:
+ ~SampleProfileLoaderBaseImpl() = default;
+ friend class SampleCoverageTracker;
+
+ inline unsigned getFunctionLoc(Function &F);
+ inline virtual ErrorOr<uint64_t> getInstWeight(const Instruction &Inst);
+ inline ErrorOr<uint64_t> getInstWeightImpl(const Instruction &Inst);
+ inline ErrorOr<uint64_t> getBlockWeight(const BasicBlock *BB);
+ mutable DenseMap<const DILocation *, const FunctionSamples *>
+ DILocation2SampleMap;
+ inline virtual const FunctionSamples *
+ findFunctionSamples(const Instruction &I) const;
+ inline void printEdgeWeight(raw_ostream &OS, Edge E);
+ inline void printBlockWeight(raw_ostream &OS, const BasicBlock *BB) const;
+ inline void printBlockEquivalence(raw_ostream &OS, const BasicBlock *BB);
+ inline bool computeBlockWeights(Function &F);
+ inline void findEquivalenceClasses(Function &F);
+ template <bool IsPostDom>
+ inline void
+ findEquivalencesFor(BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants,
+ DominatorTreeBase<BasicBlock, IsPostDom> *DomTree);
+
+ inline void propagateWeights(Function &F);
+ inline uint64_t visitEdge(Edge E, unsigned *NumUnknownEdges,
+ Edge *UnknownEdge);
+ inline void buildEdges(Function &F);
+ inline bool propagateThroughEdges(Function &F, bool UpdateBlockCount);
+ inline void clearFunctionData();
+ inline void computeDominanceAndLoopInfo(Function &F);
+ inline bool
+ computeAndPropagateWeights(Function &F,
+ const DenseSet<GlobalValue::GUID> &InlinedGUIDs);
+ inline void emitCoverageRemarks(Function &F);
+
+ /// Map basic blocks to their computed weights.
+ ///
+ /// The weight of a basic block is defined to be the maximum
+ /// of all the instruction weights in that block.
+ BlockWeightMap BlockWeights;
+
+ /// Map edges to their computed weights.
+ ///
+ /// Edge weights are computed by propagating basic block weights in
+ /// SampleProfile::propagateWeights.
+ EdgeWeightMap EdgeWeights;
+
+ /// Set of visited blocks during propagation.
+ SmallPtrSet<const BasicBlock *, 32> VisitedBlocks;
+
+ /// Set of visited edges during propagation.
+ SmallSet<Edge, 32> VisitedEdges;
+
+ /// Equivalence classes for block weights.
+ ///
+ /// Two blocks BB1 and BB2 are in the same equivalence class if they
+ /// dominate and post-dominate each other, and they are in the same loop
+ /// nest. When this happens, the two blocks are guaranteed to execute
+ /// the same number of times.
+ EquivalenceClassMap EquivalenceClass;
+
+ /// Dominance, post-dominance and loop information.
+ std::unique_ptr<DominatorTree> DT;
+ std::unique_ptr<PostDominatorTree> PDT;
+ std::unique_ptr<LoopInfo> LI;
+
+ /// Predecessors for each basic block in the CFG.
+ BlockEdgeMap Predecessors;
+
+ /// Successors for each basic block in the CFG.
+ BlockEdgeMap Successors;
+
+ /// Profile coverage tracker.
+ SampleCoverageTracker CoverageTracker;
+
+ /// Profile reader object.
+ std::unique_ptr<SampleProfileReader> Reader;
+
+ /// Samples collected for the body of this function.
+ FunctionSamples *Samples = nullptr;
+
+ /// Name of the profile file to load.
+ std::string Filename;
+
+ /// Profile Summary Info computed from sample profile.
+ ProfileSummaryInfo *PSI = nullptr;
+
+ /// Optimization Remark Emitter used to emit diagnostic remarks.
+ OptimizationRemarkEmitter *ORE = nullptr;
+};
+
+/// Clear all the per-function data used to load samples and propagate weights.
+void SampleProfileLoaderBaseImpl::clearFunctionData() {
+ BlockWeights.clear();
+ EdgeWeights.clear();
+ VisitedBlocks.clear();
+ VisitedEdges.clear();
+ EquivalenceClass.clear();
+ DT = nullptr;
+ PDT = nullptr;
+ LI = nullptr;
+ Predecessors.clear();
+ Successors.clear();
+ CoverageTracker.clear();
+}
+
+#ifndef NDEBUG
+/// Print the weight of edge \p E on stream \p OS.
+///
+/// \param OS Stream to emit the output to.
+/// \param E Edge to print.
+void SampleProfileLoaderBaseImpl::printEdgeWeight(raw_ostream &OS, Edge E) {
+ OS << "weight[" << E.first->getName() << "->" << E.second->getName()
+ << "]: " << EdgeWeights[E] << "\n";
+}
+
+/// Print the equivalence class of block \p BB on stream \p OS.
+///
+/// \param OS Stream to emit the output to.
+/// \param BB Block to print.
+void SampleProfileLoaderBaseImpl::printBlockEquivalence(raw_ostream &OS,
+ const BasicBlock *BB) {
+ const BasicBlock *Equiv = EquivalenceClass[BB];
+ OS << "equivalence[" << BB->getName()
+ << "]: " << ((Equiv) ? EquivalenceClass[BB]->getName() : "NONE") << "\n";
+}
+
+/// Print the weight of block \p BB on stream \p OS.
+///
+/// \param OS Stream to emit the output to.
+/// \param BB Block to print.
+void SampleProfileLoaderBaseImpl::printBlockWeight(raw_ostream &OS,
+ const BasicBlock *BB) const {
+ const auto &I = BlockWeights.find(BB);
+ uint64_t W = (I == BlockWeights.end() ? 0 : I->second);
+ OS << "weight[" << BB->getName() << "]: " << W << "\n";
+}
+#endif
+
+/// Get the weight for an instruction.
+///
+/// The "weight" of an instruction \p Inst is the number of samples
+/// collected on that instruction at runtime. To retrieve it, we
+/// need to compute the line number of \p Inst relative to the start of its
+/// function. We use HeaderLineno to compute the offset. We then
+/// look up the samples collected for \p Inst using BodySamples.
+///
+/// \param Inst Instruction to query.
+///
+/// \returns the weight of \p Inst.
+ErrorOr<uint64_t>
+SampleProfileLoaderBaseImpl::getInstWeight(const Instruction &Inst) {
+ return getInstWeightImpl(Inst);
+}
+
+ErrorOr<uint64_t>
+SampleProfileLoaderBaseImpl::getInstWeightImpl(const Instruction &Inst) {
+ const FunctionSamples *FS = findFunctionSamples(Inst);
+ if (!FS)
+ return std::error_code();
+
+ const DebugLoc &DLoc = Inst.getDebugLoc();
+ if (!DLoc)
+ return std::error_code();
+
+ const DILocation *DIL = DLoc;
+ uint32_t LineOffset = FunctionSamples::getOffset(DIL);
+ uint32_t Discriminator = DIL->getBaseDiscriminator();
+ ErrorOr<uint64_t> R = FS->findSamplesAt(LineOffset, Discriminator);
+ if (R) {
+ bool FirstMark =
+ CoverageTracker.markSamplesUsed(FS, LineOffset, Discriminator, R.get());
+ if (FirstMark) {
+ ORE->emit([&]() {
+ OptimizationRemarkAnalysis Remark(DEBUG_TYPE, "AppliedSamples", &Inst);
+ Remark << "Applied " << ore::NV("NumSamples", *R);
+ Remark << " samples from profile (offset: ";
+ Remark << ore::NV("LineOffset", LineOffset);
+ if (Discriminator) {
+ Remark << ".";
+ Remark << ore::NV("Discriminator", Discriminator);
+ }
+ Remark << ")";
+ return Remark;
+ });
+ }
+ LLVM_DEBUG(dbgs() << " " << DLoc.getLine() << "."
+ << DIL->getBaseDiscriminator() << ":" << Inst
+ << " (line offset: " << LineOffset << "."
+ << DIL->getBaseDiscriminator() << " - weight: " << R.get()
+ << ")\n");
+ }
+ return R;
+}
+
+/// Compute the weight of a basic block.
+///
+/// The weight of basic block \p BB is the maximum weight of all the
+/// instructions in BB.
+///
+/// \param BB The basic block to query.
+///
+/// \returns the weight for \p BB.
+ErrorOr<uint64_t>
+SampleProfileLoaderBaseImpl::getBlockWeight(const BasicBlock *BB) {
+ uint64_t Max = 0;
+ bool HasWeight = false;
+ for (auto &I : BB->getInstList()) {
+ const ErrorOr<uint64_t> &R = getInstWeight(I);
+ if (R) {
+ Max = std::max(Max, R.get());
+ HasWeight = true;
+ }
+ }
+ return HasWeight ? ErrorOr<uint64_t>(Max) : std::error_code();
+}
+
+/// Compute and store the weights of every basic block.
+///
+/// This populates the BlockWeights map by computing
+/// the weights of every basic block in the CFG.
+///
+/// \param F The function to query.
+bool SampleProfileLoaderBaseImpl::computeBlockWeights(Function &F) {
+ bool Changed = false;
+ LLVM_DEBUG(dbgs() << "Block weights\n");
+ for (const auto &BB : F) {
+ ErrorOr<uint64_t> Weight = getBlockWeight(&BB);
+ if (Weight) {
+ BlockWeights[&BB] = Weight.get();
+ VisitedBlocks.insert(&BB);
+ Changed = true;
+ }
+ LLVM_DEBUG(printBlockWeight(dbgs(), &BB));
+ }
+
+ return Changed;
+}
+
+/// Get the FunctionSamples for an instruction.
+///
+/// The FunctionSamples of an instruction \p Inst is the inlined instance
+/// in which that instruction is coming from. We traverse the inline stack
+/// of that instruction, and match it with the tree nodes in the profile.
+///
+/// \param Inst Instruction to query.
+///
+/// \returns the FunctionSamples pointer to the inlined instance.
+const FunctionSamples *SampleProfileLoaderBaseImpl::findFunctionSamples(
+ const Instruction &Inst) const {
+ const DILocation *DIL = Inst.getDebugLoc();
+ if (!DIL)
+ return Samples;
+
+ auto it = DILocation2SampleMap.try_emplace(DIL, nullptr);
+ if (it.second) {
+ it.first->second = Samples->findFunctionSamples(DIL, Reader->getRemapper());
+ }
+ return it.first->second;
+}
+
+/// Find equivalence classes for the given block.
+///
+/// This finds all the blocks that are guaranteed to execute the same
+/// number of times as \p BB1. To do this, it traverses all the
+/// descendants of \p BB1 in the dominator or post-dominator tree.
+///
+/// A block BB2 will be in the same equivalence class as \p BB1 if
+/// the following holds:
+///
+/// 1- \p BB1 is a descendant of BB2 in the opposite tree. So, if BB2
+/// is a descendant of \p BB1 in the dominator tree, then BB2 should
+/// dominate BB1 in the post-dominator tree.
+///
+/// 2- Both BB2 and \p BB1 must be in the same loop.
+///
+/// For every block BB2 that meets those two requirements, we set BB2's
+/// equivalence class to \p BB1.
+///
+/// \param BB1 Block to check.
+/// \param Descendants Descendants of \p BB1 in either the dom or pdom tree.
+/// \param DomTree Opposite dominator tree. If \p Descendants is filled
+/// with blocks from \p BB1's dominator tree, then
+/// this is the post-dominator tree, and vice versa.
+template <bool IsPostDom>
+void SampleProfileLoaderBaseImpl::findEquivalencesFor(
+ BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants,
+ DominatorTreeBase<BasicBlock, IsPostDom> *DomTree) {
+ const BasicBlock *EC = EquivalenceClass[BB1];
+ uint64_t Weight = BlockWeights[EC];
+ for (const auto *BB2 : Descendants) {
+ bool IsDomParent = DomTree->dominates(BB2, BB1);
+ bool IsInSameLoop = LI->getLoopFor(BB1) == LI->getLoopFor(BB2);
+ if (BB1 != BB2 && IsDomParent && IsInSameLoop) {
+ EquivalenceClass[BB2] = EC;
+ // If BB2 is visited, then the entire EC should be marked as visited.
+ if (VisitedBlocks.count(BB2)) {
+ VisitedBlocks.insert(EC);
+ }
+
+ // If BB2 is heavier than BB1, make BB2 have the same weight
+ // as BB1.
+ //
+ // Note that we don't worry about the opposite situation here
+ // (when BB2 is lighter than BB1). We will deal with this
+ // during the propagation phase. Right now, we just want to
+ // make sure that BB1 has the largest weight of all the
+ // members of its equivalence set.
+ Weight = std::max(Weight, BlockWeights[BB2]);
+ }
+ }
+ if (EC == &EC->getParent()->getEntryBlock()) {
+ BlockWeights[EC] = Samples->getHeadSamples() + 1;
+ } else {
+ BlockWeights[EC] = Weight;
+ }
+}
+
+/// Find equivalence classes.
+///
+/// Since samples may be missing from blocks, we can fill in the gaps by setting
+/// the weights of all the blocks in the same equivalence class to the same
+/// weight. To compute the concept of equivalence, we use dominance and loop
+/// information. Two blocks B1 and B2 are in the same equivalence class if B1
+/// dominates B2, B2 post-dominates B1 and both are in the same loop.
+///
+/// \param F The function to query.
+void SampleProfileLoaderBaseImpl::findEquivalenceClasses(Function &F) {
+ SmallVector<BasicBlock *, 8> DominatedBBs;
+ LLVM_DEBUG(dbgs() << "\nBlock equivalence classes\n");
+ // Find equivalence sets based on dominance and post-dominance information.
+ for (auto &BB : F) {
+ BasicBlock *BB1 = &BB;
+
+ // Compute BB1's equivalence class once.
+ if (EquivalenceClass.count(BB1)) {
+ LLVM_DEBUG(printBlockEquivalence(dbgs(), BB1));
+ continue;
+ }
+
+ // By default, blocks are in their own equivalence class.
+ EquivalenceClass[BB1] = BB1;
+
+ // Traverse all the blocks dominated by BB1. We are looking for
+ // every basic block BB2 such that:
+ //
+ // 1- BB1 dominates BB2.
+ // 2- BB2 post-dominates BB1.
+ // 3- BB1 and BB2 are in the same loop nest.
+ //
+ // If all those conditions hold, it means that BB2 is executed
+ // as many times as BB1, so they are placed in the same equivalence
+ // class by making BB2's equivalence class be BB1.
+ DominatedBBs.clear();
+ DT->getDescendants(BB1, DominatedBBs);
+ findEquivalencesFor(BB1, DominatedBBs, PDT.get());
+
+ LLVM_DEBUG(printBlockEquivalence(dbgs(), BB1));
+ }
+
+ // Assign weights to equivalence classes.
+ //
+ // All the basic blocks in the same equivalence class will execute
+ // the same number of times. Since we know that the head block in
+ // each equivalence class has the largest weight, assign that weight
+ // to all the blocks in that equivalence class.
+ LLVM_DEBUG(
+ dbgs() << "\nAssign the same weight to all blocks in the same class\n");
+ for (auto &BI : F) {
+ const BasicBlock *BB = &BI;
+ const BasicBlock *EquivBB = EquivalenceClass[BB];
+ if (BB != EquivBB)
+ BlockWeights[BB] = BlockWeights[EquivBB];
+ LLVM_DEBUG(printBlockWeight(dbgs(), BB));
+ }
+}
+
+/// Visit the given edge to decide if it has a valid weight.
+///
+/// If \p E has not been visited before, we copy to \p UnknownEdge
+/// and increment the count of unknown edges.
+///
+/// \param E Edge to visit.
+/// \param NumUnknownEdges Current number of unknown edges.
+/// \param UnknownEdge Set if E has not been visited before.
+///
+/// \returns E's weight, if known. Otherwise, return 0.
+uint64_t SampleProfileLoaderBaseImpl::visitEdge(Edge E,
+ unsigned *NumUnknownEdges,
+ Edge *UnknownEdge) {
+ if (!VisitedEdges.count(E)) {
+ (*NumUnknownEdges)++;
+ *UnknownEdge = E;
+ return 0;
+ }
+
+ return EdgeWeights[E];
+}
+
+/// Propagate weights through incoming/outgoing edges.
+///
+/// If the weight of a basic block is known, and there is only one edge
+/// with an unknown weight, we can calculate the weight of that edge.
+///
+/// Similarly, if all the edges have a known count, we can calculate the
+/// count of the basic block, if needed.
+///
+/// \param F Function to process.
+/// \param UpdateBlockCount Whether we should update basic block counts that
+/// has already been annotated.
+///
+/// \returns True if new weights were assigned to edges or blocks.
+bool SampleProfileLoaderBaseImpl::propagateThroughEdges(Function &F,
+ bool UpdateBlockCount) {
+ bool Changed = false;
+ LLVM_DEBUG(dbgs() << "\nPropagation through edges\n");
+ for (const auto &BI : F) {
+ const BasicBlock *BB = &BI;
+ const BasicBlock *EC = EquivalenceClass[BB];
+
+ // Visit all the predecessor and successor edges to determine
+ // which ones have a weight assigned already. Note that it doesn't
+ // matter that we only keep track of a single unknown edge. The
+ // only case we are interested in handling is when only a single
+ // edge is unknown (see setEdgeOrBlockWeight).
+ for (unsigned i = 0; i < 2; i++) {
+ uint64_t TotalWeight = 0;
+ unsigned NumUnknownEdges = 0, NumTotalEdges = 0;
+ Edge UnknownEdge, SelfReferentialEdge, SingleEdge;
+
+ if (i == 0) {
+ // First, visit all predecessor edges.
+ NumTotalEdges = Predecessors[BB].size();
+ for (auto *Pred : Predecessors[BB]) {
+ Edge E = std::make_pair(Pred, BB);
+ TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
+ if (E.first == E.second)
+ SelfReferentialEdge = E;
+ }
+ if (NumTotalEdges == 1) {
+ SingleEdge = std::make_pair(Predecessors[BB][0], BB);
+ }
+ } else {
+ // On the second round, visit all successor edges.
+ NumTotalEdges = Successors[BB].size();
+ for (auto *Succ : Successors[BB]) {
+ Edge E = std::make_pair(BB, Succ);
+ TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
+ }
+ if (NumTotalEdges == 1) {
+ SingleEdge = std::make_pair(BB, Successors[BB][0]);
+ }
+ }
+
+ // After visiting all the edges, there are three cases that we
+ // can handle immediately:
+ //
+ // - All the edge weights are known (i.e., NumUnknownEdges == 0).
+ // In this case, we simply check that the sum of all the edges
+ // is the same as BB's weight. If not, we change BB's weight
+ // to match. Additionally, if BB had not been visited before,
+ // we mark it visited.
+ //
+ // - Only one edge is unknown and BB has already been visited.
+ // In this case, we can compute the weight of the edge by
+ // subtracting the total block weight from all the known
+ // edge weights. If the edges weight more than BB, then the
+ // edge of the last remaining edge is set to zero.
+ //
+ // - There exists a self-referential edge and the weight of BB is
+ // known. In this case, this edge can be based on BB's weight.
+ // We add up all the other known edges and set the weight on
+ // the self-referential edge as we did in the previous case.
+ //
+ // In any other case, we must continue iterating. Eventually,
+ // all edges will get a weight, or iteration will stop when
+ // it reaches SampleProfileMaxPropagateIterations.
+ if (NumUnknownEdges <= 1) {
+ uint64_t &BBWeight = BlockWeights[EC];
+ if (NumUnknownEdges == 0) {
+ if (!VisitedBlocks.count(EC)) {
+ // If we already know the weight of all edges, the weight of the
+ // basic block can be computed. It should be no larger than the sum
+ // of all edge weights.
+ if (TotalWeight > BBWeight) {
+ BBWeight = TotalWeight;
+ Changed = true;
+ LLVM_DEBUG(dbgs() << "All edge weights for " << BB->getName()
+ << " known. Set weight for block: ";
+ printBlockWeight(dbgs(), BB););
+ }
+ } else if (NumTotalEdges == 1 &&
+ EdgeWeights[SingleEdge] < BlockWeights[EC]) {
+ // If there is only one edge for the visited basic block, use the
+ // block weight to adjust edge weight if edge weight is smaller.
+ EdgeWeights[SingleEdge] = BlockWeights[EC];
+ Changed = true;
+ }
+ } else if (NumUnknownEdges == 1 && VisitedBlocks.count(EC)) {
+ // If there is a single unknown edge and the block has been
+ // visited, then we can compute E's weight.
+ if (BBWeight >= TotalWeight)
+ EdgeWeights[UnknownEdge] = BBWeight - TotalWeight;
+ else
+ EdgeWeights[UnknownEdge] = 0;
+ const BasicBlock *OtherEC;
+ if (i == 0)
+ OtherEC = EquivalenceClass[UnknownEdge.first];
+ else
+ OtherEC = EquivalenceClass[UnknownEdge.second];
+ // Edge weights should never exceed the BB weights it connects.
+ if (VisitedBlocks.count(OtherEC) &&
+ EdgeWeights[UnknownEdge] > BlockWeights[OtherEC])
+ EdgeWeights[UnknownEdge] = BlockWeights[OtherEC];
+ VisitedEdges.insert(UnknownEdge);
+ Changed = true;
+ LLVM_DEBUG(dbgs() << "Set weight for edge: ";
+ printEdgeWeight(dbgs(), UnknownEdge));
+ }
+ } else if (VisitedBlocks.count(EC) && BlockWeights[EC] == 0) {
+ // If a block Weights 0, all its in/out edges should weight 0.
+ if (i == 0) {
+ for (auto *Pred : Predecessors[BB]) {
+ Edge E = std::make_pair(Pred, BB);
+ EdgeWeights[E] = 0;
+ VisitedEdges.insert(E);
+ }
+ } else {
+ for (auto *Succ : Successors[BB]) {
+ Edge E = std::make_pair(BB, Succ);
+ EdgeWeights[E] = 0;
+ VisitedEdges.insert(E);
+ }
+ }
+ } else if (SelfReferentialEdge.first && VisitedBlocks.count(EC)) {
+ uint64_t &BBWeight = BlockWeights[BB];
+ // We have a self-referential edge and the weight of BB is known.
+ if (BBWeight >= TotalWeight)
+ EdgeWeights[SelfReferentialEdge] = BBWeight - TotalWeight;
+ else
+ EdgeWeights[SelfReferentialEdge] = 0;
+ VisitedEdges.insert(SelfReferentialEdge);
+ Changed = true;
+ LLVM_DEBUG(dbgs() << "Set self-referential edge weight to: ";
+ printEdgeWeight(dbgs(), SelfReferentialEdge));
+ }
+ if (UpdateBlockCount && !VisitedBlocks.count(EC) && TotalWeight > 0) {
+ BlockWeights[EC] = TotalWeight;
+ VisitedBlocks.insert(EC);
+ Changed = true;
+ }
+ }
+ }
+
+ return Changed;
+}
+
+/// Build in/out edge lists for each basic block in the CFG.
+///
+/// We are interested in unique edges. If a block B1 has multiple
+/// edges to another block B2, we only add a single B1->B2 edge.
+void SampleProfileLoaderBaseImpl::buildEdges(Function &F) {
+ for (auto &BI : F) {
+ BasicBlock *B1 = &BI;
+
+ // Add predecessors for B1.
+ SmallPtrSet<BasicBlock *, 16> Visited;
+ if (!Predecessors[B1].empty())
+ llvm_unreachable("Found a stale predecessors list in a basic block.");
+ for (BasicBlock *B2 : predecessors(B1))
+ if (Visited.insert(B2).second)
+ Predecessors[B1].push_back(B2);
+
+ // Add successors for B1.
+ Visited.clear();
+ if (!Successors[B1].empty())
+ llvm_unreachable("Found a stale successors list in a basic block.");
+ for (BasicBlock *B2 : successors(B1))
+ if (Visited.insert(B2).second)
+ Successors[B1].push_back(B2);
+ }
+}
+
+/// Propagate weights into edges
+///
+/// The following rules are applied to every block BB in the CFG:
+///
+/// - If BB has a single predecessor/successor, then the weight
+/// of that edge is the weight of the block.
+///
+/// - If all incoming or outgoing edges are known except one, and the
+/// weight of the block is already known, the weight of the unknown
+/// edge will be the weight of the block minus the sum of all the known
+/// edges. If the sum of all the known edges is larger than BB's weight,
+/// we set the unknown edge weight to zero.
+///
+/// - If there is a self-referential edge, and the weight of the block is
+/// known, the weight for that edge is set to the weight of the block
+/// minus the weight of the other incoming edges to that block (if
+/// known).
+void SampleProfileLoaderBaseImpl::propagateWeights(Function &F) {
+ bool Changed = true;
+ unsigned I = 0;
+
+ // If BB weight is larger than its corresponding loop's header BB weight,
+ // use the BB weight to replace the loop header BB weight.
+ for (auto &BI : F) {
+ BasicBlock *BB = &BI;
+ Loop *L = LI->getLoopFor(BB);
+ if (!L) {
+ continue;
+ }
+ BasicBlock *Header = L->getHeader();
+ if (Header && BlockWeights[BB] > BlockWeights[Header]) {
+ BlockWeights[Header] = BlockWeights[BB];
+ }
+ }
+
+ // Before propagation starts, build, for each block, a list of
+ // unique predecessors and successors. This is necessary to handle
+ // identical edges in multiway branches. Since we visit all blocks and all
+ // edges of the CFG, it is cleaner to build these lists once at the start
+ // of the pass.
+ buildEdges(F);
+
+ // Propagate until we converge or we go past the iteration limit.
+ while (Changed && I++ < SampleProfileMaxPropagateIterations) {
+ Changed = propagateThroughEdges(F, false);
+ }
+
+ // The first propagation propagates BB counts from annotated BBs to unknown
+ // BBs. The 2nd propagation pass resets edges weights, and use all BB weights
+ // to propagate edge weights.
+ VisitedEdges.clear();
+ Changed = true;
+ while (Changed && I++ < SampleProfileMaxPropagateIterations) {
+ Changed = propagateThroughEdges(F, false);
+ }
+
+ // The 3rd propagation pass allows adjust annotated BB weights that are
+ // obviously wrong.
+ Changed = true;
+ while (Changed && I++ < SampleProfileMaxPropagateIterations) {
+ Changed = propagateThroughEdges(F, true);
+ }
+}
+
+/// Generate branch weight metadata for all branches in \p F.
+///
+/// Branch weights are computed out of instruction samples using a
+/// propagation heuristic. Propagation proceeds in 3 phases:
+///
+/// 1- Assignment of block weights. All the basic blocks in the function
+/// are initial assigned the same weight as their most frequently
+/// executed instruction.
+///
+/// 2- Creation of equivalence classes. Since samples may be missing from
+/// blocks, we can fill in the gaps by setting the weights of all the
+/// blocks in the same equivalence class to the same weight. To compute
+/// the concept of equivalence, we use dominance and loop information.
+/// Two blocks B1 and B2 are in the same equivalence class if B1
+/// dominates B2, B2 post-dominates B1 and both are in the same loop.
+///
+/// 3- Propagation of block weights into edges. This uses a simple
+/// propagation heuristic. The following rules are applied to every
+/// block BB in the CFG:
+///
+/// - If BB has a single predecessor/successor, then the weight
+/// of that edge is the weight of the block.
+///
+/// - If all the edges are known except one, and the weight of the
+/// block is already known, the weight of the unknown edge will
+/// be the weight of the block minus the sum of all the known
+/// edges. If the sum of all the known edges is larger than BB's weight,
+/// we set the unknown edge weight to zero.
+///
+/// - If there is a self-referential edge, and the weight of the block is
+/// known, the weight for that edge is set to the weight of the block
+/// minus the weight of the other incoming edges to that block (if
+/// known).
+///
+/// Since this propagation is not guaranteed to finalize for every CFG, we
+/// only allow it to proceed for a limited number of iterations (controlled
+/// by -sample-profile-max-propagate-iterations).
+///
+/// FIXME: Try to replace this propagation heuristic with a scheme
+/// that is guaranteed to finalize. A work-list approach similar to
+/// the standard value propagation algorithm used by SSA-CCP might
+/// work here.
+///
+/// \param F The function to query.
+///
+/// \returns true if \p F was modified. Returns false, otherwise.
+bool SampleProfileLoaderBaseImpl::computeAndPropagateWeights(
+ Function &F, const DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
+ bool Changed = (InlinedGUIDs.size() != 0);
+
+ // Compute basic block weights.
+ Changed |= computeBlockWeights(F);
+
+ if (Changed) {
+ // Add an entry count to the function using the samples gathered at the
+ // function entry.
+ // Sets the GUIDs that are inlined in the profiled binary. This is used
+ // for ThinLink to make correct liveness analysis, and also make the IR
+ // match the profiled binary before annotation.
+ F.setEntryCount(
+ ProfileCount(Samples->getHeadSamples() + 1, Function::PCT_Real),
+ &InlinedGUIDs);
+
+ // Compute dominance and loop info needed for propagation.
+ computeDominanceAndLoopInfo(F);
+
+ // Find equivalence classes.
+ findEquivalenceClasses(F);
+
+ // Propagate weights to all edges.
+ propagateWeights(F);
+ }
+
+ return Changed;
+}
+
+void SampleProfileLoaderBaseImpl::emitCoverageRemarks(Function &F) {
+ // If coverage checking was requested, compute it now.
+ if (SampleProfileRecordCoverage) {
+ unsigned Used = CoverageTracker.countUsedRecords(Samples, PSI);
+ unsigned Total = CoverageTracker.countBodyRecords(Samples, PSI);
+ unsigned Coverage = CoverageTracker.computeCoverage(Used, Total);
+ if (Coverage < SampleProfileRecordCoverage) {
+ F.getContext().diagnose(DiagnosticInfoSampleProfile(
+ F.getSubprogram()->getFilename(), getFunctionLoc(F),
+ Twine(Used) + " of " + Twine(Total) + " available profile records (" +
+ Twine(Coverage) + "%) were applied",
+ DS_Warning));
+ }
+ }
+
+ if (SampleProfileSampleCoverage) {
+ uint64_t Used = CoverageTracker.getTotalUsedSamples();
+ uint64_t Total = CoverageTracker.countBodySamples(Samples, PSI);
+ unsigned Coverage = CoverageTracker.computeCoverage(Used, Total);
+ if (Coverage < SampleProfileSampleCoverage) {
+ F.getContext().diagnose(DiagnosticInfoSampleProfile(
+ F.getSubprogram()->getFilename(), getFunctionLoc(F),
+ Twine(Used) + " of " + Twine(Total) + " available profile samples (" +
+ Twine(Coverage) + "%) were applied",
+ DS_Warning));
+ }
+ }
+}
+
+/// Get the line number for the function header.
+///
+/// This looks up function \p F in the current compilation unit and
+/// retrieves the line number where the function is defined. This is
+/// line 0 for all the samples read from the profile file. Every line
+/// number is relative to this line.
+///
+/// \param F Function object to query.
+///
+/// \returns the line number where \p F is defined. If it returns 0,
+/// it means that there is no debug information available for \p F.
+unsigned SampleProfileLoaderBaseImpl::getFunctionLoc(Function &F) {
+ if (DISubprogram *S = F.getSubprogram())
+ return S->getLine();
+
+ if (NoWarnSampleUnused)
+ return 0;
+
+ // If the start of \p F is missing, emit a diagnostic to inform the user
+ // about the missed opportunity.
+ F.getContext().diagnose(DiagnosticInfoSampleProfile(
+ "No debug information found in function " + F.getName() +
+ ": Function profile not used",
+ DS_Warning));
+ return 0;
+}
+
+void SampleProfileLoaderBaseImpl::computeDominanceAndLoopInfo(Function &F) {
+ DT.reset(new DominatorTree);
+ DT->recalculate(F);
+
+ PDT.reset(new PostDominatorTree(F));
+
+ LI.reset(new LoopInfo);
+ LI->analyze(*DT);
+}
+
+#undef DEBUG_TYPE
+
+} // namespace llvm
+#endif // LLVM_TRANSFORMS_UTILS_SAMPLEPROFILELOADERIMPL_H
diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h
new file mode 100644
index 000000000000..ed93133076db
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h
@@ -0,0 +1,98 @@
+////===- SampleProfileLoadBaseUtil.h - Profile loader util func --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file provides the utility functions for the sampled PGO loader base
+/// implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_SAMPLEPROFILELOADERUTIL_H
+#define LLVM_TRANSFORMS_UTILS_SAMPLEPROFILELOADERUTIL_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/DebugLoc.h"
+#include "llvm/IR/Function.h"
+#include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/CommandLine.h"
+
+namespace llvm {
+using namespace sampleprof;
+
+class ProfileSummaryInfo;
+
+extern cl::opt<unsigned> SampleProfileMaxPropagateIterations;
+extern cl::opt<unsigned> SampleProfileRecordCoverage;
+extern cl::opt<unsigned> SampleProfileSampleCoverage;
+extern cl::opt<bool> NoWarnSampleUnused;
+
+namespace sampleprofutil {
+
+class SampleCoverageTracker {
+public:
+ bool markSamplesUsed(const FunctionSamples *FS, uint32_t LineOffset,
+ uint32_t Discriminator, uint64_t Samples);
+ unsigned computeCoverage(unsigned Used, unsigned Total) const;
+ unsigned countUsedRecords(const FunctionSamples *FS,
+ ProfileSummaryInfo *PSI) const;
+ unsigned countBodyRecords(const FunctionSamples *FS,
+ ProfileSummaryInfo *PSI) const;
+ uint64_t getTotalUsedSamples() const { return TotalUsedSamples; }
+ uint64_t countBodySamples(const FunctionSamples *FS,
+ ProfileSummaryInfo *PSI) const;
+
+ void clear() {
+ SampleCoverage.clear();
+ TotalUsedSamples = 0;
+ }
+ void setProfAccForSymsInList(bool V) { ProfAccForSymsInList = V; }
+
+private:
+ using BodySampleCoverageMap = std::map<LineLocation, unsigned>;
+ using FunctionSamplesCoverageMap =
+ DenseMap<const FunctionSamples *, BodySampleCoverageMap>;
+
+ /// Coverage map for sampling records.
+ ///
+ /// This map keeps a record of sampling records that have been matched to
+ /// an IR instruction. This is used to detect some form of staleness in
+ /// profiles (see flag -sample-profile-check-coverage).
+ ///
+ /// Each entry in the map corresponds to a FunctionSamples instance. This is
+ /// another map that counts how many times the sample record at the
+ /// given location has been used.
+ FunctionSamplesCoverageMap SampleCoverage;
+
+ /// Number of samples used from the profile.
+ ///
+ /// When a sampling record is used for the first time, the samples from
+ /// that record are added to this accumulator. Coverage is later computed
+ /// based on the total number of samples available in this function and
+ /// its callsites.
+ ///
+ /// Note that this accumulator tracks samples used from a single function
+ /// and all the inlined callsites. Strictly, we should have a map of counters
+ /// keyed by FunctionSamples pointers, but these stats are cleared after
+ /// every function, so we just need to keep a single counter.
+ uint64_t TotalUsedSamples = 0;
+
+ // For symbol in profile symbol list, whether to regard their profiles
+ // to be accurate. This is passed from the SampleLoader instance.
+ bool ProfAccForSymsInList = false;
+};
+
+/// Return true if the given callsite is hot wrt to hot cutoff threshold.
+bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI,
+ bool ProfAccForSymsInList);
+} // end of namespace sampleprofutil
+} // end of namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_SAMPLEPROFILELOADERUTIL_H
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 812700ac25da..528a7166db55 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -82,6 +82,8 @@
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h"
+#include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
@@ -97,6 +99,7 @@
using namespace llvm;
using namespace sampleprof;
+using namespace llvm::sampleprofutil;
using ProfileCount = Function::ProfileCount;
#define DEBUG_TYPE "sample-profile"
#define CSINLINE_DEBUG DEBUG_TYPE "-inline"
@@ -132,26 +135,6 @@ static cl::opt<std::string> SampleProfileRemappingFile(
"sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"),
cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden);
-static cl::opt<unsigned> SampleProfileMaxPropagateIterations(
- "sample-profile-max-propagate-iterations", cl::init(100),
- cl::desc("Maximum number of iterations to go through when propagating "
- "sample block/edge weights through the CFG."));
-
-static cl::opt<unsigned> SampleProfileRecordCoverage(
- "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"),
- cl::desc("Emit a warning if less than N% of records in the input profile "
- "are matched to the IR."));
-
-static cl::opt<unsigned> SampleProfileSampleCoverage(
- "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"),
- cl::desc("Emit a warning if less than N% of samples in the input profile "
- "are matched to the IR."));
-
-static cl::opt<bool> NoWarnSampleUnused(
- "no-warn-sample-unused", cl::init(false), cl::Hidden,
- cl::desc("Use this option to turn off/on warnings about function with "
- "samples but without debug information to use those samples. "));
-
static cl::opt<bool> ProfileSampleAccurate(
"profile-sample-accurate", cl::Hidden, cl::init(false),
cl::desc("If the sample profile is accurate, we will mark all un-sampled "
@@ -244,65 +227,12 @@ using EdgeWeightMap = DenseMap<Edge, uint64_t>;
using BlockEdgeMap =
DenseMap<const BasicBlock *, SmallVector<const BasicBlock *, 8>>;
-class SampleCoverageTracker {
-public:
- bool markSamplesUsed(const FunctionSamples *FS, uint32_t LineOffset,
- uint32_t Discriminator, uint64_t Samples);
- unsigned computeCoverage(unsigned Used, unsigned Total) const;
- unsigned countUsedRecords(const FunctionSamples *FS,
- ProfileSummaryInfo *PSI) const;
- unsigned countBodyRecords(const FunctionSamples *FS,
- ProfileSummaryInfo *PSI) const;
- uint64_t getTotalUsedSamples() const { return TotalUsedSamples; }
- uint64_t countBodySamples(const FunctionSamples *FS,
- ProfileSummaryInfo *PSI) const;
-
- void clear() {
- SampleCoverage.clear();
- TotalUsedSamples = 0;
- }
- inline void setProfAccForSymsInList(bool V) { ProfAccForSymsInList = V; }
-
-private:
- using BodySampleCoverageMap = std::map<LineLocation, unsigned>;
- using FunctionSamplesCoverageMap =
- DenseMap<const FunctionSamples *, BodySampleCoverageMap>;
-
- /// Coverage map for sampling records.
- ///
- /// This map keeps a record of sampling records that have been matched to
- /// an IR instruction. This is used to detect some form of staleness in
- /// profiles (see flag -sample-profile-check-coverage).
- ///
- /// Each entry in the map corresponds to a FunctionSamples instance. This is
- /// another map that counts how many times the sample record at the
- /// given location has been used.
- FunctionSamplesCoverageMap SampleCoverage;
-
- /// Number of samples used from the profile.
- ///
- /// When a sampling record is used for the first time, the samples from
- /// that record are added to this accumulator. Coverage is later computed
- /// based on the total number of samples available in this function and
- /// its callsites.
- ///
- /// Note that this accumulator tracks samples used from a single function
- /// and all the inlined callsites. Strictly, we should have a map of counters
- /// keyed by FunctionSamples pointers, but these stats are cleared after
- /// every function, so we just need to keep a single counter.
- uint64_t TotalUsedSamples = 0;
-
- // For symbol in profile symbol list, whether to regard their profiles
- // to be accurate. This is passed from the SampleLoader instance.
- bool ProfAccForSymsInList = false;
-};
-
class GUIDToFuncNameMapper {
public:
GUIDToFuncNameMapper(Module &M, SampleProfileReader &Reader,
- DenseMap<uint64_t, StringRef> &GUIDToFuncNameMap)
+ DenseMap<uint64_t, StringRef> &GUIDToFuncNameMap)
: CurrentReader(Reader), CurrentModule(M),
- CurrentGUIDToFuncNameMap(GUIDToFuncNameMap) {
+ CurrentGUIDToFuncNameMap(GUIDToFuncNameMap) {
if (!CurrentReader.useMD5())
return;
@@ -397,99 +327,6 @@ using CandidateQueue =
PriorityQueue<InlineCandidate, std::vector<InlineCandidate>,
CandidateComparer>;
-class SampleProfileLoaderBaseImpl {
-public:
- SampleProfileLoaderBaseImpl(std::string Name) : Filename(Name) {}
- void dump() { Reader->dump(); }
-
-protected:
- friend class SampleCoverageTracker;
-
- ~SampleProfileLoaderBaseImpl() = default;
-
- unsigned getFunctionLoc(Function &F);
- virtual ErrorOr<uint64_t> getInstWeight(const Instruction &Inst);
- ErrorOr<uint64_t> getInstWeightImpl(const Instruction &Inst);
- ErrorOr<uint64_t> getBlockWeight(const BasicBlock *BB);
- mutable DenseMap<const DILocation *, const FunctionSamples *>
- DILocation2SampleMap;
- virtual const FunctionSamples *
- findFunctionSamples(const Instruction &I) const;
- void printEdgeWeight(raw_ostream &OS, Edge E);
- void printBlockWeight(raw_ostream &OS, const BasicBlock *BB) const;
- void printBlockEquivalence(raw_ostream &OS, const BasicBlock *BB);
- bool computeBlockWeights(Function &F);
- void findEquivalenceClasses(Function &F);
- template <bool IsPostDom>
- void findEquivalencesFor(BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants,
- DominatorTreeBase<BasicBlock, IsPostDom> *DomTree);
-
- void propagateWeights(Function &F);
- uint64_t visitEdge(Edge E, unsigned *NumUnknownEdges, Edge *UnknownEdge);
- void buildEdges(Function &F);
- bool propagateThroughEdges(Function &F, bool UpdateBlockCount);
- void clearFunctionData();
- void computeDominanceAndLoopInfo(Function &F);
- bool
- computeAndPropagateWeights(Function &F,
- const DenseSet<GlobalValue::GUID> &InlinedGUIDs);
- void emitCoverageRemarks(Function &F);
-
- /// Map basic blocks to their computed weights.
- ///
- /// The weight of a basic block is defined to be the maximum
- /// of all the instruction weights in that block.
- BlockWeightMap BlockWeights;
-
- /// Map edges to their computed weights.
- ///
- /// Edge weights are computed by propagating basic block weights in
- /// SampleProfile::propagateWeights.
- EdgeWeightMap EdgeWeights;
-
- /// Set of visited blocks during propagation.
- SmallPtrSet<const BasicBlock *, 32> VisitedBlocks;
-
- /// Set of visited edges during propagation.
- SmallSet<Edge, 32> VisitedEdges;
-
- /// Equivalence classes for block weights.
- ///
- /// Two blocks BB1 and BB2 are in the same equivalence class if they
- /// dominate and post-dominate each other, and they are in the same loop
- /// nest. When this happens, the two blocks are guaranteed to execute
- /// the same number of times.
- EquivalenceClassMap EquivalenceClass;
-
- /// Dominance, post-dominance and loop information.
- std::unique_ptr<DominatorTree> DT;
- std::unique_ptr<PostDominatorTree> PDT;
- std::unique_ptr<LoopInfo> LI;
-
- /// Predecessors for each basic block in the CFG.
- BlockEdgeMap Predecessors;
-
- /// Successors for each basic block in the CFG.
- BlockEdgeMap Successors;
-
- SampleCoverageTracker CoverageTracker;
-
- /// Profile reader object.
- std::unique_ptr<SampleProfileReader> Reader;
-
- /// Samples collected for the body of this function.
- FunctionSamples *Samples = nullptr;
-
- /// Name of the profile file to load.
- std::string Filename;
-
- /// Profile Summary Info computed from sample profile.
- ProfileSummaryInfo *PSI = nullptr;
-
- /// Optimization Remark Emitter used to emit diagnostic remarks.
- OptimizationRemarkEmitter *ORE = nullptr;
-};
-
/// Sample profile pass.
///
/// This pass reads profile data from the file specified by
@@ -663,193 +500,6 @@ class SampleProfileLoaderLegacyPass : public ModulePass {
} // end anonymous namespace
-/// Return true if the given callsite is hot wrt to hot cutoff threshold.
-///
-/// Functions that were inlined in the original binary will be represented
-/// in the inline stack in the sample profile. If the profile shows that
-/// the original inline decision was "good" (i.e., the callsite is executed
-/// frequently), then we will recreate the inline decision and apply the
-/// profile from the inlined callsite.
-///
-/// To decide whether an inlined callsite is hot, we compare the callsite
-/// sample count with the hot cutoff computed by ProfileSummaryInfo, it is
-/// regarded as hot if the count is above the cutoff value.
-///
-/// When ProfileAccurateForSymsInList is enabled and profile symbol list
-/// is present, functions in the profile symbol list but without profile will
-/// be regarded as cold and much less inlining will happen in CGSCC inlining
-/// pass, so we tend to lower the hot criteria here to allow more early
-/// inlining to happen for warm callsites and it is helpful for performance.
-static bool callsiteIsHot(const FunctionSamples *CallsiteFS,
- ProfileSummaryInfo *PSI, bool ProfAccForSymsInList) {
- if (!CallsiteFS)
- return false; // The callsite was not inlined in the original binary.
-
- assert(PSI && "PSI is expected to be non null");
- uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
- if (ProfAccForSymsInList)
- return !PSI->isColdCount(CallsiteTotalSamples);
- else
- return PSI->isHotCount(CallsiteTotalSamples);
-}
-
-/// Mark as used the sample record for the given function samples at
-/// (LineOffset, Discriminator).
-///
-/// \returns true if this is the first time we mark the given record.
-bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS,
- uint32_t LineOffset,
- uint32_t Discriminator,
- uint64_t Samples) {
- LineLocation Loc(LineOffset, Discriminator);
- unsigned &Count = SampleCoverage[FS][Loc];
- bool FirstTime = (++Count == 1);
- if (FirstTime)
- TotalUsedSamples += Samples;
- return FirstTime;
-}
-
-/// Return the number of sample records that were applied from this profile.
-///
-/// This count does not include records from cold inlined callsites.
-unsigned
-SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS,
- ProfileSummaryInfo *PSI) const {
- auto I = SampleCoverage.find(FS);
-
- // The size of the coverage map for FS represents the number of records
- // that were marked used at least once.
- unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0;
-
- // If there are inlined callsites in this function, count the samples found
- // in the respective bodies. However, do not bother counting callees with 0
- // total samples, these are callees that were never invoked at runtime.
- for (const auto &I : FS->getCallsiteSamples())
- for (const auto &J : I.second) {
- const FunctionSamples *CalleeSamples = &J.second;
- if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
- Count += countUsedRecords(CalleeSamples, PSI);
- }
-
- return Count;
-}
-
-/// Return the number of sample records in the body of this profile.
-///
-/// This count does not include records from cold inlined callsites.
-unsigned
-SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS,
- ProfileSummaryInfo *PSI) const {
- unsigned Count = FS->getBodySamples().size();
-
- // Only count records in hot callsites.
- for (const auto &I : FS->getCallsiteSamples())
- for (const auto &J : I.second) {
- const FunctionSamples *CalleeSamples = &J.second;
- if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
- Count += countBodyRecords(CalleeSamples, PSI);
- }
-
- return Count;
-}
-
-/// Return the number of samples collected in the body of this profile.
-///
-/// This count does not include samples from cold inlined callsites.
-uint64_t
-SampleCoverageTracker::countBodySamples(const FunctionSamples *FS,
- ProfileSummaryInfo *PSI) const {
- uint64_t Total = 0;
- for (const auto &I : FS->getBodySamples())
- Total += I.second.getSamples();
-
- // Only count samples in hot callsites.
- for (const auto &I : FS->getCallsiteSamples())
- for (const auto &J : I.second) {
- const FunctionSamples *CalleeSamples = &J.second;
- if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
- Total += countBodySamples(CalleeSamples, PSI);
- }
-
- return Total;
-}
-
-/// Return the fraction of sample records used in this profile.
-///
-/// The returned value is an unsigned integer in the range 0-100 indicating
-/// the percentage of sample records that were used while applying this
-/// profile to the associated function.
-unsigned SampleCoverageTracker::computeCoverage(unsigned Used,
- unsigned Total) const {
- assert(Used <= Total &&
- "number of used records cannot exceed the total number of records");
- return Total > 0 ? Used * 100 / Total : 100;
-}
-
-/// Clear all the per-function data used to load samples and propagate weights.
-void SampleProfileLoaderBaseImpl::clearFunctionData() {
- BlockWeights.clear();
- EdgeWeights.clear();
- VisitedBlocks.clear();
- VisitedEdges.clear();
- EquivalenceClass.clear();
- DT = nullptr;
- PDT = nullptr;
- LI = nullptr;
- Predecessors.clear();
- Successors.clear();
- CoverageTracker.clear();
-}
-
-#ifndef NDEBUG
-/// Print the weight of edge \p E on stream \p OS.
-///
-/// \param OS Stream to emit the output to.
-/// \param E Edge to print.
-void SampleProfileLoaderBaseImpl::printEdgeWeight(raw_ostream &OS, Edge E) {
- OS << "weight[" << E.first->getName() << "->" << E.second->getName()
- << "]: " << EdgeWeights[E] << "\n";
-}
-
-/// Print the equivalence class of block \p BB on stream \p OS.
-///
-/// \param OS Stream to emit the output to.
-/// \param BB Block to print.
-void SampleProfileLoaderBaseImpl::printBlockEquivalence(raw_ostream &OS,
- const BasicBlock *BB) {
- const BasicBlock *Equiv = EquivalenceClass[BB];
- OS << "equivalence[" << BB->getName()
- << "]: " << ((Equiv) ? EquivalenceClass[BB]->getName() : "NONE") << "\n";
-}
-
-/// Print the weight of block \p BB on stream \p OS.
-///
-/// \param OS Stream to emit the output to.
-/// \param BB Block to print.
-void SampleProfileLoaderBaseImpl::printBlockWeight(raw_ostream &OS,
- const BasicBlock *BB) const {
- const auto &I = BlockWeights.find(BB);
- uint64_t W = (I == BlockWeights.end() ? 0 : I->second);
- OS << "weight[" << BB->getName() << "]: " << W << "\n";
-}
-#endif
-
-/// Get the weight for an instruction.
-///
-/// The "weight" of an instruction \p Inst is the number of samples
-/// collected on that instruction at runtime. To retrieve it, we
-/// need to compute the line number of \p Inst relative to the start of its
-/// function. We use HeaderLineno to compute the offset. We then
-/// look up the samples collected for \p Inst using BodySamples.
-///
-/// \param Inst Instruction to query.
-///
-/// \returns the weight of \p Inst.
-ErrorOr<uint64_t>
-SampleProfileLoaderBaseImpl::getInstWeight(const Instruction &Inst) {
- return getInstWeightImpl(Inst);
-}
-
ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
if (FunctionSamples::ProfileIsProbeBased)
return getProbeWeight(Inst);
@@ -876,46 +526,6 @@ ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
return getInstWeightImpl(Inst);
}
-ErrorOr<uint64_t>
-SampleProfileLoaderBaseImpl::getInstWeightImpl(const Instruction &Inst) {
- const FunctionSamples *FS = findFunctionSamples(Inst);
- if (!FS)
- return std::error_code();
-
- const DebugLoc &DLoc = Inst.getDebugLoc();
- if (!DLoc)
- return std::error_code();
-
- const DILocation *DIL = DLoc;
- uint32_t LineOffset = FunctionSamples::getOffset(DIL);
- uint32_t Discriminator = DIL->getBaseDiscriminator();
- ErrorOr<uint64_t> R = FS->findSamplesAt(LineOffset, Discriminator);
- if (R) {
- bool FirstMark =
- CoverageTracker.markSamplesUsed(FS, LineOffset, Discriminator, R.get());
- if (FirstMark) {
- ORE->emit([&]() {
- OptimizationRemarkAnalysis Remark(DEBUG_TYPE, "AppliedSamples", &Inst);
- Remark << "Applied " << ore::NV("NumSamples", *R);
- Remark << " samples from profile (offset: ";
- Remark << ore::NV("LineOffset", LineOffset);
- if (Discriminator) {
- Remark << ".";
- Remark << ore::NV("Discriminator", Discriminator);
- }
- Remark << ")";
- return Remark;
- });
- }
- LLVM_DEBUG(dbgs() << " " << DLoc.getLine() << "."
- << DIL->getBaseDiscriminator() << ":" << Inst
- << " (line offset: " << LineOffset << "."
- << DIL->getBaseDiscriminator() << " - weight: " << R.get()
- << ")\n");
- }
- return R;
-}
-
ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) {
assert(FunctionSamples::ProfileIsProbeBased &&
"Profile is not pseudo probe based");
@@ -961,50 +571,6 @@ ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) {
return R;
}
-/// Compute the weight of a basic block.
-///
-/// The weight of basic block \p BB is the maximum weight of all the
-/// instructions in BB.
-///
-/// \param BB The basic block to query.
-///
-/// \returns the weight for \p BB.
-ErrorOr<uint64_t>
-SampleProfileLoaderBaseImpl::getBlockWeight(const BasicBlock *BB) {
- uint64_t Max = 0;
- bool HasWeight = false;
- for (auto &I : BB->getInstList()) {
- const ErrorOr<uint64_t> &R = getInstWeight(I);
- if (R) {
- Max = std::max(Max, R.get());
- HasWeight = true;
- }
- }
- return HasWeight ? ErrorOr<uint64_t>(Max) : std::error_code();
-}
-
-/// Compute and store the weights of every basic block.
-///
-/// This populates the BlockWeights map by computing
-/// the weights of every basic block in the CFG.
-///
-/// \param F The function to query.
-bool SampleProfileLoaderBaseImpl::computeBlockWeights(Function &F) {
- bool Changed = false;
- LLVM_DEBUG(dbgs() << "Block weights\n");
- for (const auto &BB : F) {
- ErrorOr<uint64_t> Weight = getBlockWeight(&BB);
- if (Weight) {
- BlockWeights[&BB] = Weight.get();
- VisitedBlocks.insert(&BB);
- Changed = true;
- }
- LLVM_DEBUG(printBlockWeight(dbgs(), &BB));
- }
-
- return Changed;
-}
-
/// Get the FunctionSamples for a call instruction.
///
/// The FunctionSamples of a call/invoke instruction \p Inst is the inlined
@@ -1099,28 +665,6 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
return R;
}
-/// Get the FunctionSamples for an instruction.
-///
-/// The FunctionSamples of an instruction \p Inst is the inlined instance
-/// in which that instruction is coming from. We traverse the inline stack
-/// of that instruction, and match it with the tree nodes in the profile.
-///
-/// \param Inst Instruction to query.
-///
-/// \returns the FunctionSamples pointer to the inlined instance.
-const FunctionSamples *SampleProfileLoaderBaseImpl::findFunctionSamples(
- const Instruction &Inst) const {
- const DILocation *DIL = Inst.getDebugLoc();
- if (!DIL)
- return Samples;
-
- auto it = DILocation2SampleMap.try_emplace(DIL, nullptr);
- if (it.second) {
- it.first->second = Samples->findFunctionSamples(DIL, Reader->getRemapper());
- }
- return it.first->second;
-}
-
const FunctionSamples *
SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
if (FunctionSamples::ProfileIsProbeBased) {
@@ -1656,507 +1200,17 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
return Changed;
}
-/// Find equivalence classes for the given block.
-///
-/// This finds all the blocks that are guaranteed to execute the same
-/// number of times as \p BB1. To do this, it traverses all the
-/// descendants of \p BB1 in the dominator or post-dominator tree.
-///
-/// A block BB2 will be in the same equivalence class as \p BB1 if
-/// the following holds:
-///
-/// 1- \p BB1 is a descendant of BB2 in the opposite tree. So, if BB2
-/// is a descendant of \p BB1 in the dominator tree, then BB2 should
-/// dominate BB1 in the post-dominator tree.
-///
-/// 2- Both BB2 and \p BB1 must be in the same loop.
-///
-/// For every block BB2 that meets those two requirements, we set BB2's
-/// equivalence class to \p BB1.
-///
-/// \param BB1 Block to check.
-/// \param Descendants Descendants of \p BB1 in either the dom or pdom tree.
-/// \param DomTree Opposite dominator tree. If \p Descendants is filled
-/// with blocks from \p BB1's dominator tree, then
-/// this is the post-dominator tree, and vice versa.
-template <bool IsPostDom>
-void SampleProfileLoaderBaseImpl::findEquivalencesFor(
- BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants,
- DominatorTreeBase<BasicBlock, IsPostDom> *DomTree) {
- const BasicBlock *EC = EquivalenceClass[BB1];
- uint64_t Weight = BlockWeights[EC];
- for (const auto *BB2 : Descendants) {
- bool IsDomParent = DomTree->dominates(BB2, BB1);
- bool IsInSameLoop = LI->getLoopFor(BB1) == LI->getLoopFor(BB2);
- if (BB1 != BB2 && IsDomParent && IsInSameLoop) {
- EquivalenceClass[BB2] = EC;
- // If BB2 is visited, then the entire EC should be marked as visited.
- if (VisitedBlocks.count(BB2)) {
- VisitedBlocks.insert(EC);
- }
-
- // If BB2 is heavier than BB1, make BB2 have the same weight
- // as BB1.
- //
- // Note that we don't worry about the opposite situation here
- // (when BB2 is lighter than BB1). We will deal with this
- // during the propagation phase. Right now, we just want to
- // make sure that BB1 has the largest weight of all the
- // members of its equivalence set.
- Weight = std::max(Weight, BlockWeights[BB2]);
- }
- }
- if (EC == &EC->getParent()->getEntryBlock()) {
- BlockWeights[EC] = Samples->getHeadSamples() + 1;
- } else {
- BlockWeights[EC] = Weight;
- }
-}
-
-/// Find equivalence classes.
-///
-/// Since samples may be missing from blocks, we can fill in the gaps by setting
-/// the weights of all the blocks in the same equivalence class to the same
-/// weight. To compute the concept of equivalence, we use dominance and loop
-/// information. Two blocks B1 and B2 are in the same equivalence class if B1
-/// dominates B2, B2 post-dominates B1 and both are in the same loop.
-///
-/// \param F The function to query.
-void SampleProfileLoaderBaseImpl::findEquivalenceClasses(Function &F) {
- SmallVector<BasicBlock *, 8> DominatedBBs;
- LLVM_DEBUG(dbgs() << "\nBlock equivalence classes\n");
- // Find equivalence sets based on dominance and post-dominance information.
- for (auto &BB : F) {
- BasicBlock *BB1 = &BB;
-
- // Compute BB1's equivalence class once.
- if (EquivalenceClass.count(BB1)) {
- LLVM_DEBUG(printBlockEquivalence(dbgs(), BB1));
- continue;
- }
-
- // By default, blocks are in their own equivalence class.
- EquivalenceClass[BB1] = BB1;
-
- // Traverse all the blocks dominated by BB1. We are looking for
- // every basic block BB2 such that:
- //
- // 1- BB1 dominates BB2.
- // 2- BB2 post-dominates BB1.
- // 3- BB1 and BB2 are in the same loop nest.
- //
- // If all those conditions hold, it means that BB2 is executed
- // as many times as BB1, so they are placed in the same equivalence
- // class by making BB2's equivalence class be BB1.
- DominatedBBs.clear();
- DT->getDescendants(BB1, DominatedBBs);
- findEquivalencesFor(BB1, DominatedBBs, PDT.get());
-
- LLVM_DEBUG(printBlockEquivalence(dbgs(), BB1));
- }
-
- // Assign weights to equivalence classes.
- //
- // All the basic blocks in the same equivalence class will execute
- // the same number of times. Since we know that the head block in
- // each equivalence class has the largest weight, assign that weight
- // to all the blocks in that equivalence class.
- LLVM_DEBUG(
- dbgs() << "\nAssign the same weight to all blocks in the same class\n");
- for (auto &BI : F) {
- const BasicBlock *BB = &BI;
- const BasicBlock *EquivBB = EquivalenceClass[BB];
- if (BB != EquivBB)
- BlockWeights[BB] = BlockWeights[EquivBB];
- LLVM_DEBUG(printBlockWeight(dbgs(), BB));
- }
-}
-
-/// Visit the given edge to decide if it has a valid weight.
-///
-/// If \p E has not been visited before, we copy to \p UnknownEdge
-/// and increment the count of unknown edges.
-///
-/// \param E Edge to visit.
-/// \param NumUnknownEdges Current number of unknown edges.
-/// \param UnknownEdge Set if E has not been visited before.
-///
-/// \returns E's weight, if known. Otherwise, return 0.
-uint64_t SampleProfileLoaderBaseImpl::visitEdge(Edge E,
- unsigned *NumUnknownEdges,
- Edge *UnknownEdge) {
- if (!VisitedEdges.count(E)) {
- (*NumUnknownEdges)++;
- *UnknownEdge = E;
- return 0;
- }
-
- return EdgeWeights[E];
-}
-
-/// Propagate weights through incoming/outgoing edges.
-///
-/// If the weight of a basic block is known, and there is only one edge
-/// with an unknown weight, we can calculate the weight of that edge.
-///
-/// Similarly, if all the edges have a known count, we can calculate the
-/// count of the basic block, if needed.
-///
-/// \param F Function to process.
-/// \param UpdateBlockCount Whether we should update basic block counts that
-/// has already been annotated.
-///
-/// \returns True if new weights were assigned to edges or blocks.
-bool SampleProfileLoaderBaseImpl::propagateThroughEdges(Function &F,
- bool UpdateBlockCount) {
- bool Changed = false;
- LLVM_DEBUG(dbgs() << "\nPropagation through edges\n");
- for (const auto &BI : F) {
- const BasicBlock *BB = &BI;
- const BasicBlock *EC = EquivalenceClass[BB];
-
- // Visit all the predecessor and successor edges to determine
- // which ones have a weight assigned already. Note that it doesn't
- // matter that we only keep track of a single unknown edge. The
- // only case we are interested in handling is when only a single
- // edge is unknown (see setEdgeOrBlockWeight).
- for (unsigned i = 0; i < 2; i++) {
- uint64_t TotalWeight = 0;
- unsigned NumUnknownEdges = 0, NumTotalEdges = 0;
- Edge UnknownEdge, SelfReferentialEdge, SingleEdge;
-
- if (i == 0) {
- // First, visit all predecessor edges.
- NumTotalEdges = Predecessors[BB].size();
- for (auto *Pred : Predecessors[BB]) {
- Edge E = std::make_pair(Pred, BB);
- TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
- if (E.first == E.second)
- SelfReferentialEdge = E;
- }
- if (NumTotalEdges == 1) {
- SingleEdge = std::make_pair(Predecessors[BB][0], BB);
- }
- } else {
- // On the second round, visit all successor edges.
- NumTotalEdges = Successors[BB].size();
- for (auto *Succ : Successors[BB]) {
- Edge E = std::make_pair(BB, Succ);
- TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
- }
- if (NumTotalEdges == 1) {
- SingleEdge = std::make_pair(BB, Successors[BB][0]);
- }
- }
-
- // After visiting all the edges, there are three cases that we
- // can handle immediately:
- //
- // - All the edge weights are known (i.e., NumUnknownEdges == 0).
- // In this case, we simply check that the sum of all the edges
- // is the same as BB's weight. If not, we change BB's weight
- // to match. Additionally, if BB had not been visited before,
- // we mark it visited.
- //
- // - Only one edge is unknown and BB has already been visited.
- // In this case, we can compute the weight of the edge by
- // subtracting the total block weight from all the known
- // edge weights. If the edges weight more than BB, then the
- // edge of the last remaining edge is set to zero.
- //
- // - There exists a self-referential edge and the weight of BB is
- // known. In this case, this edge can be based on BB's weight.
- // We add up all the other known edges and set the weight on
- // the self-referential edge as we did in the previous case.
- //
- // In any other case, we must continue iterating. Eventually,
- // all edges will get a weight, or iteration will stop when
- // it reaches SampleProfileMaxPropagateIterations.
- if (NumUnknownEdges <= 1) {
- uint64_t &BBWeight = BlockWeights[EC];
- if (NumUnknownEdges == 0) {
- if (!VisitedBlocks.count(EC)) {
- // If we already know the weight of all edges, the weight of the
- // basic block can be computed. It should be no larger than the sum
- // of all edge weights.
- if (TotalWeight > BBWeight) {
- BBWeight = TotalWeight;
- Changed = true;
- LLVM_DEBUG(dbgs() << "All edge weights for " << BB->getName()
- << " known. Set weight for block: ";
- printBlockWeight(dbgs(), BB););
- }
- } else if (NumTotalEdges == 1 &&
- EdgeWeights[SingleEdge] < BlockWeights[EC]) {
- // If there is only one edge for the visited basic block, use the
- // block weight to adjust edge weight if edge weight is smaller.
- EdgeWeights[SingleEdge] = BlockWeights[EC];
- Changed = true;
- }
- } else if (NumUnknownEdges == 1 && VisitedBlocks.count(EC)) {
- // If there is a single unknown edge and the block has been
- // visited, then we can compute E's weight.
- if (BBWeight >= TotalWeight)
- EdgeWeights[UnknownEdge] = BBWeight - TotalWeight;
- else
- EdgeWeights[UnknownEdge] = 0;
- const BasicBlock *OtherEC;
- if (i == 0)
- OtherEC = EquivalenceClass[UnknownEdge.first];
- else
- OtherEC = EquivalenceClass[UnknownEdge.second];
- // Edge weights should never exceed the BB weights it connects.
- if (VisitedBlocks.count(OtherEC) &&
- EdgeWeights[UnknownEdge] > BlockWeights[OtherEC])
- EdgeWeights[UnknownEdge] = BlockWeights[OtherEC];
- VisitedEdges.insert(UnknownEdge);
- Changed = true;
- LLVM_DEBUG(dbgs() << "Set weight for edge: ";
- printEdgeWeight(dbgs(), UnknownEdge));
- }
- } else if (VisitedBlocks.count(EC) && BlockWeights[EC] == 0) {
- // If a block Weights 0, all its in/out edges should weight 0.
- if (i == 0) {
- for (auto *Pred : Predecessors[BB]) {
- Edge E = std::make_pair(Pred, BB);
- EdgeWeights[E] = 0;
- VisitedEdges.insert(E);
- }
- } else {
- for (auto *Succ : Successors[BB]) {
- Edge E = std::make_pair(BB, Succ);
- EdgeWeights[E] = 0;
- VisitedEdges.insert(E);
- }
- }
- } else if (SelfReferentialEdge.first && VisitedBlocks.count(EC)) {
- uint64_t &BBWeight = BlockWeights[BB];
- // We have a self-referential edge and the weight of BB is known.
- if (BBWeight >= TotalWeight)
- EdgeWeights[SelfReferentialEdge] = BBWeight - TotalWeight;
- else
- EdgeWeights[SelfReferentialEdge] = 0;
- VisitedEdges.insert(SelfReferentialEdge);
- Changed = true;
- LLVM_DEBUG(dbgs() << "Set self-referential edge weight to: ";
- printEdgeWeight(dbgs(), SelfReferentialEdge));
- }
- if (UpdateBlockCount && !VisitedBlocks.count(EC) && TotalWeight > 0) {
- BlockWeights[EC] = TotalWeight;
- VisitedBlocks.insert(EC);
- Changed = true;
- }
- }
- }
-
- return Changed;
-}
-
-/// Build in/out edge lists for each basic block in the CFG.
-///
-/// We are interested in unique edges. If a block B1 has multiple
-/// edges to another block B2, we only add a single B1->B2 edge.
-void SampleProfileLoaderBaseImpl::buildEdges(Function &F) {
- for (auto &BI : F) {
- BasicBlock *B1 = &BI;
-
- // Add predecessors for B1.
- SmallPtrSet<BasicBlock *, 16> Visited;
- if (!Predecessors[B1].empty())
- llvm_unreachable("Found a stale predecessors list in a basic block.");
- for (BasicBlock *B2 : predecessors(B1))
- if (Visited.insert(B2).second)
- Predecessors[B1].push_back(B2);
-
- // Add successors for B1.
- Visited.clear();
- if (!Successors[B1].empty())
- llvm_unreachable("Found a stale successors list in a basic block.");
- for (BasicBlock *B2 : successors(B1))
- if (Visited.insert(B2).second)
- Successors[B1].push_back(B2);
- }
-}
-
/// Returns the sorted CallTargetMap \p M by count in descending order.
-static SmallVector<InstrProfValueData, 2> GetSortedValueDataFromCallTargets(
- const SampleRecord::CallTargetMap & M) {
+static SmallVector<InstrProfValueData, 2>
+GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M) {
SmallVector<InstrProfValueData, 2> R;
for (const auto &I : SampleRecord::SortCallTargets(M)) {
- R.emplace_back(InstrProfValueData{FunctionSamples::getGUID(I.first), I.second});
+ R.emplace_back(
+ InstrProfValueData{FunctionSamples::getGUID(I.first), I.second});
}
return R;
}
-/// Propagate weights into edges
-///
-/// The following rules are applied to every block BB in the CFG:
-///
-/// - If BB has a single predecessor/successor, then the weight
-/// of that edge is the weight of the block.
-///
-/// - If all incoming or outgoing edges are known except one, and the
-/// weight of the block is already known, the weight of the unknown
-/// edge will be the weight of the block minus the sum of all the known
-/// edges. If the sum of all the known edges is larger than BB's weight,
-/// we set the unknown edge weight to zero.
-///
-/// - If there is a self-referential edge, and the weight of the block is
-/// known, the weight for that edge is set to the weight of the block
-/// minus the weight of the other incoming edges to that block (if
-/// known).
-void SampleProfileLoaderBaseImpl::propagateWeights(Function &F) {
- bool Changed = true;
- unsigned I = 0;
-
- // If BB weight is larger than its corresponding loop's header BB weight,
- // use the BB weight to replace the loop header BB weight.
- for (auto &BI : F) {
- BasicBlock *BB = &BI;
- Loop *L = LI->getLoopFor(BB);
- if (!L) {
- continue;
- }
- BasicBlock *Header = L->getHeader();
- if (Header && BlockWeights[BB] > BlockWeights[Header]) {
- BlockWeights[Header] = BlockWeights[BB];
- }
- }
-
- // Before propagation starts, build, for each block, a list of
- // unique predecessors and successors. This is necessary to handle
- // identical edges in multiway branches. Since we visit all blocks and all
- // edges of the CFG, it is cleaner to build these lists once at the start
- // of the pass.
- buildEdges(F);
-
- // Propagate until we converge or we go past the iteration limit.
- while (Changed && I++ < SampleProfileMaxPropagateIterations) {
- Changed = propagateThroughEdges(F, false);
- }
-
- // The first propagation propagates BB counts from annotated BBs to unknown
- // BBs. The 2nd propagation pass resets edges weights, and use all BB weights
- // to propagate edge weights.
- VisitedEdges.clear();
- Changed = true;
- while (Changed && I++ < SampleProfileMaxPropagateIterations) {
- Changed = propagateThroughEdges(F, false);
- }
-
- // The 3rd propagation pass allows adjust annotated BB weights that are
- // obviously wrong.
- Changed = true;
- while (Changed && I++ < SampleProfileMaxPropagateIterations) {
- Changed = propagateThroughEdges(F, true);
- }
-}
-
-/// Generate branch weight metadata for all branches in \p F.
-///
-/// Branch weights are computed out of instruction samples using a
-/// propagation heuristic. Propagation proceeds in 3 phases:
-///
-/// 1- Assignment of block weights. All the basic blocks in the function
-/// are initial assigned the same weight as their most frequently
-/// executed instruction.
-///
-/// 2- Creation of equivalence classes. Since samples may be missing from
-/// blocks, we can fill in the gaps by setting the weights of all the
-/// blocks in the same equivalence class to the same weight. To compute
-/// the concept of equivalence, we use dominance and loop information.
-/// Two blocks B1 and B2 are in the same equivalence class if B1
-/// dominates B2, B2 post-dominates B1 and both are in the same loop.
-///
-/// 3- Propagation of block weights into edges. This uses a simple
-/// propagation heuristic. The following rules are applied to every
-/// block BB in the CFG:
-///
-/// - If BB has a single predecessor/successor, then the weight
-/// of that edge is the weight of the block.
-///
-/// - If all the edges are known except one, and the weight of the
-/// block is already known, the weight of the unknown edge will
-/// be the weight of the block minus the sum of all the known
-/// edges. If the sum of all the known edges is larger than BB's weight,
-/// we set the unknown edge weight to zero.
-///
-/// - If there is a self-referential edge, and the weight of the block is
-/// known, the weight for that edge is set to the weight of the block
-/// minus the weight of the other incoming edges to that block (if
-/// known).
-///
-/// Since this propagation is not guaranteed to finalize for every CFG, we
-/// only allow it to proceed for a limited number of iterations (controlled
-/// by -sample-profile-max-propagate-iterations).
-///
-/// FIXME: Try to replace this propagation heuristic with a scheme
-/// that is guaranteed to finalize. A work-list approach similar to
-/// the standard value propagation algorithm used by SSA-CCP might
-/// work here.
-///
-/// \param F The function to query.
-///
-/// \returns true if \p F was modified. Returns false, otherwise.
-bool SampleProfileLoaderBaseImpl::computeAndPropagateWeights(
- Function &F, const DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
- bool Changed = (InlinedGUIDs.size() != 0);
-
- // Compute basic block weights.
- Changed |= computeBlockWeights(F);
-
- if (Changed) {
- // Add an entry count to the function using the samples gathered at the
- // function entry.
- // Sets the GUIDs that are inlined in the profiled binary. This is used
- // for ThinLink to make correct liveness analysis, and also make the IR
- // match the profiled binary before annotation.
- F.setEntryCount(
- ProfileCount(Samples->getHeadSamples() + 1, Function::PCT_Real),
- &InlinedGUIDs);
-
- // Compute dominance and loop info needed for propagation.
- computeDominanceAndLoopInfo(F);
-
- // Find equivalence classes.
- findEquivalenceClasses(F);
-
- // Propagate weights to all edges.
- propagateWeights(F);
- }
-
- return Changed;
-}
-
-void SampleProfileLoaderBaseImpl::emitCoverageRemarks(Function &F) {
- // If coverage checking was requested, compute it now.
- if (SampleProfileRecordCoverage) {
- unsigned Used = CoverageTracker.countUsedRecords(Samples, PSI);
- unsigned Total = CoverageTracker.countBodyRecords(Samples, PSI);
- unsigned Coverage = CoverageTracker.computeCoverage(Used, Total);
- if (Coverage < SampleProfileRecordCoverage) {
- F.getContext().diagnose(DiagnosticInfoSampleProfile(
- F.getSubprogram()->getFilename(), getFunctionLoc(F),
- Twine(Used) + " of " + Twine(Total) + " available profile records (" +
- Twine(Coverage) + "%) were applied",
- DS_Warning));
- }
- }
-
- if (SampleProfileSampleCoverage) {
- uint64_t Used = CoverageTracker.getTotalUsedSamples();
- uint64_t Total = CoverageTracker.countBodySamples(Samples, PSI);
- unsigned Coverage = CoverageTracker.computeCoverage(Used, Total);
- if (Coverage < SampleProfileSampleCoverage) {
- F.getContext().diagnose(DiagnosticInfoSampleProfile(
- F.getSubprogram()->getFilename(), getFunctionLoc(F),
- Twine(Used) + " of " + Twine(Total) + " available profile samples (" +
- Twine(Coverage) + "%) were applied",
- DS_Warning));
- }
- }
-}
-
// Generate MD_prof metadata for every branch instruction using the
// edge weights computed during propagation.
void SampleProfileLoader::generateMDProfMetadata(Function &F) {
@@ -2264,43 +1318,6 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
}
}
-/// Get the line number for the function header.
-///
-/// This looks up function \p F in the current compilation unit and
-/// retrieves the line number where the function is defined. This is
-/// line 0 for all the samples read from the profile file. Every line
-/// number is relative to this line.
-///
-/// \param F Function object to query.
-///
-/// \returns the line number where \p F is defined. If it returns 0,
-/// it means that there is no debug information available for \p F.
-unsigned SampleProfileLoaderBaseImpl::getFunctionLoc(Function &F) {
- if (DISubprogram *S = F.getSubprogram())
- return S->getLine();
-
- if (NoWarnSampleUnused)
- return 0;
-
- // If the start of \p F is missing, emit a diagnostic to inform the user
- // about the missed opportunity.
- F.getContext().diagnose(DiagnosticInfoSampleProfile(
- "No debug information found in function " + F.getName() +
- ": Function profile not used",
- DS_Warning));
- return 0;
-}
-
-void SampleProfileLoaderBaseImpl::computeDominanceAndLoopInfo(Function &F) {
- DT.reset(new DominatorTree);
- DT->recalculate(F);
-
- PDT.reset(new PostDominatorTree(F));
-
- LI.reset(new LoopInfo);
- LI->analyze(*DT);
-}
-
/// Once all the branch weights are computed, we emit the MD_prof
/// metadata on BB using the computed values for each of its branches.
///
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 30c9a3daa90b..dc6d21f9e8f0 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -57,6 +57,7 @@ add_llvm_component_library(LLVMTransformUtils
StripGCRelocates.cpp
SSAUpdater.cpp
SSAUpdaterBulk.cpp
+ SampleProfileLoaderBaseUtil.cpp
SanitizerStats.cpp
SimplifyCFG.cpp
SimplifyIndVar.cpp
diff --git a/llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp b/llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp
new file mode 100644
index 000000000000..d2cd0deb3496
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp
@@ -0,0 +1,163 @@
+//===- SampleProfileLoaderBaseUtil.cpp - Profile loader Util func ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the SampleProfileLoader base utility functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
+
+namespace llvm {
+
+cl::opt<unsigned> SampleProfileMaxPropagateIterations(
+ "sample-profile-max-propagate-iterations", cl::init(100),
+ cl::desc("Maximum number of iterations to go through when propagating "
+ "sample block/edge weights through the CFG."));
+
+cl::opt<unsigned> SampleProfileRecordCoverage(
+ "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"),
+ cl::desc("Emit a warning if less than N% of records in the input profile "
+ "are matched to the IR."));
+
+cl::opt<unsigned> SampleProfileSampleCoverage(
+ "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"),
+ cl::desc("Emit a warning if less than N% of samples in the input profile "
+ "are matched to the IR."));
+
+cl::opt<bool> NoWarnSampleUnused(
+ "no-warn-sample-unused", cl::init(false), cl::Hidden,
+ cl::desc("Use this option to turn off/on warnings about function with "
+ "samples but without debug information to use those samples. "));
+
+namespace sampleprofutil {
+
+/// Return true if the given callsite is hot wrt to hot cutoff threshold.
+///
+/// Functions that were inlined in the original binary will be represented
+/// in the inline stack in the sample profile. If the profile shows that
+/// the original inline decision was "good" (i.e., the callsite is executed
+/// frequently), then we will recreate the inline decision and apply the
+/// profile from the inlined callsite.
+///
+/// To decide whether an inlined callsite is hot, we compare the callsite
+/// sample count with the hot cutoff computed by ProfileSummaryInfo, it is
+/// regarded as hot if the count is above the cutoff value.
+///
+/// When ProfileAccurateForSymsInList is enabled and profile symbol list
+/// is present, functions in the profile symbol list but without profile will
+/// be regarded as cold and much less inlining will happen in CGSCC inlining
+/// pass, so we tend to lower the hot criteria here to allow more early
+/// inlining to happen for warm callsites and it is helpful for performance.
+bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI,
+ bool ProfAccForSymsInList) {
+ if (!CallsiteFS)
+ return false; // The callsite was not inlined in the original binary.
+
+ assert(PSI && "PSI is expected to be non null");
+ uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
+ if (ProfAccForSymsInList)
+ return !PSI->isColdCount(CallsiteTotalSamples);
+ else
+ return PSI->isHotCount(CallsiteTotalSamples);
+}
+
+/// Mark as used the sample record for the given function samples at
+/// (LineOffset, Discriminator).
+///
+/// \returns true if this is the first time we mark the given record.
+bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS,
+ uint32_t LineOffset,
+ uint32_t Discriminator,
+ uint64_t Samples) {
+ LineLocation Loc(LineOffset, Discriminator);
+ unsigned &Count = SampleCoverage[FS][Loc];
+ bool FirstTime = (++Count == 1);
+ if (FirstTime)
+ TotalUsedSamples += Samples;
+ return FirstTime;
+}
+
+/// Return the number of sample records that were applied from this profile.
+///
+/// This count does not include records from cold inlined callsites.
+unsigned
+SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS,
+ ProfileSummaryInfo *PSI) const {
+ auto I = SampleCoverage.find(FS);
+
+ // The size of the coverage map for FS represents the number of records
+ // that were marked used at least once.
+ unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0;
+
+ // If there are inlined callsites in this function, count the samples found
+ // in the respective bodies. However, do not bother counting callees with 0
+ // total samples, these are callees that were never invoked at runtime.
+ for (const auto &I : FS->getCallsiteSamples())
+ for (const auto &J : I.second) {
+ const FunctionSamples *CalleeSamples = &J.second;
+ if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
+ Count += countUsedRecords(CalleeSamples, PSI);
+ }
+
+ return Count;
+}
+
+/// Return the number of sample records in the body of this profile.
+///
+/// This count does not include records from cold inlined callsites.
+unsigned
+SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS,
+ ProfileSummaryInfo *PSI) const {
+ unsigned Count = FS->getBodySamples().size();
+
+ // Only count records in hot callsites.
+ for (const auto &I : FS->getCallsiteSamples())
+ for (const auto &J : I.second) {
+ const FunctionSamples *CalleeSamples = &J.second;
+ if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
+ Count += countBodyRecords(CalleeSamples, PSI);
+ }
+
+ return Count;
+}
+
+/// Return the number of samples collected in the body of this profile.
+///
+/// This count does not include samples from cold inlined callsites.
+uint64_t
+SampleCoverageTracker::countBodySamples(const FunctionSamples *FS,
+ ProfileSummaryInfo *PSI) const {
+ uint64_t Total = 0;
+ for (const auto &I : FS->getBodySamples())
+ Total += I.second.getSamples();
+
+ // Only count samples in hot callsites.
+ for (const auto &I : FS->getCallsiteSamples())
+ for (const auto &J : I.second) {
+ const FunctionSamples *CalleeSamples = &J.second;
+ if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
+ Total += countBodySamples(CalleeSamples, PSI);
+ }
+
+ return Total;
+}
+
+/// Return the fraction of sample records used in this profile.
+///
+/// The returned value is an unsigned integer in the range 0-100 indicating
+/// the percentage of sample records that were used while applying this
+/// profile to the associated function.
+unsigned SampleCoverageTracker::computeCoverage(unsigned Used,
+ unsigned Total) const {
+ assert(Used <= Total &&
+ "number of used records cannot exceed the total number of records");
+ return Total > 0 ? Used * 100 / Total : 100;
+}
+
+} // end of namespace sampleprofutil
+} // end of namespace llvm
More information about the llvm-commits
mailing list