[llvm] [BOLT] Report flow conservation scores (PR #127954)
Amir Ayupov via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 27 13:22:09 PST 2025
================
@@ -0,0 +1,577 @@
+//===- bolt/Passes/ProfileQualityStats.cpp ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the profile quality stats calculation pass.
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Passes/ProfileQualityStats.h"
+#include "bolt/Core/BinaryBasicBlock.h"
+#include "bolt/Core/BinaryFunction.h"
+#include "bolt/Utils/CommandLineOpts.h"
+#include "llvm/Support/CommandLine.h"
+#include <queue>
+#include <unordered_map>
+#include <unordered_set>
+
+using namespace llvm;
+using namespace bolt;
+
+namespace opts {
+extern cl::opt<unsigned> Verbosity;
+cl::opt<unsigned> TopFunctionsForProfileQualityCheck(
+ "top-functions-for-profile-quality-check",
+ cl::desc("number of hottest functions to print aggregated "
+ "profile quality stats of."),
+ cl::init(1000), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltOptCategory));
+cl::opt<unsigned> PercentileForProfileQualityCheck(
+ "percentile-for-profile-quality-check",
+ cl::desc("Percentile of profile quality distributions over hottest "
+ "functions to report."),
+ cl::init(95), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltOptCategory));
+} // namespace opts
+
+namespace {
+using FunctionListType = std::vector<const BinaryFunction *>;
+using function_iterator = FunctionListType::iterator;
+
+// Function number -> vector of flows for BBs in the function
+using TotalFlowMapTy = std::unordered_map<uint64_t, std::vector<uint64_t>>;
+// Function number -> flow count
+using FunctionFlowMapTy = std::unordered_map<uint64_t, uint64_t>;
+struct FlowInfo {
+ TotalFlowMapTy TotalIncomingFlows;
+ TotalFlowMapTy TotalOutgoingFlows;
+ TotalFlowMapTy TotalMaxCountMaps;
+ TotalFlowMapTy TotalMinCountMaps;
+ FunctionFlowMapTy CallGraphIncomingFlows;
+};
+
+template <typename T>
+void printDistribution(raw_ostream &OS, std::vector<T> &values,
+ bool Fraction = false) {
+ // Assume values are sorted.
+ if (values.empty())
+ return;
+
+ OS << " Length : " << values.size() << "\n";
+
+ auto printLine = [&](std::string Text, double Percent) {
+ int Rank = int(values.size() * (100 - Percent) / 100);
+ if (Percent == 0)
+ Rank = values.size() - 1;
+ if (Fraction)
+ OS << " " << Text << std::string(11 - Text.length(), ' ') << ": "
+ << formatv("{0:P}", values[Rank]) << "\n";
+ else
+ OS << " " << Text << std::string(11 - Text.length(), ' ') << ": "
+ << values[Rank] << "\n";
+ };
+
+ printLine("MAX", 0);
+ const int percentages[] = {1, 5, 10, 20, 50, 80};
+ for (size_t i = 0; i < sizeof(percentages) / sizeof(percentages[0]); ++i) {
+ printLine("TOP " + std::to_string(percentages[i]) + "%", percentages[i]);
+ }
+ printLine("MIN", 100);
+}
+
+void printCFGContinuityStats(raw_ostream &OS,
+ iterator_range<function_iterator> &Functions) {
+ // Given a perfect profile, every positive-execution-count BB should be
+ // connected to an entry of the function through a positive-execution-count
+ // directed path in the control flow graph.
+ std::vector<size_t> NumUnreachables;
+ std::vector<size_t> SumECUnreachables;
+ std::vector<double> FractionECUnreachables;
+
+ for (const BinaryFunction *Function : Functions) {
+ if (Function->size() <= 1)
+ continue;
+
+ // Compute the sum of all BB execution counts (ECs).
+ size_t NumPosECBBs = 0;
+ size_t SumAllBBEC = 0;
+ for (const BinaryBasicBlock &BB : *Function) {
+ const size_t BBEC = BB.getKnownExecutionCount();
+ NumPosECBBs += !!BBEC;
+ SumAllBBEC += BBEC;
+ }
+
+ // Perform BFS on subgraph of CFG induced by positive weight edges.
+ // Compute the number of BBs reachable from the entry(s) of the function and
+ // the sum of their execution counts (ECs).
+ std::unordered_set<unsigned> Visited;
+ std::queue<unsigned> Queue;
+ size_t SumReachableBBEC = 0;
+
+ Function->forEachEntryPoint([&](uint64_t Offset, const MCSymbol *Label) {
+ const BinaryBasicBlock *EntryBB = Function->getBasicBlockAtOffset(Offset);
+ if (!EntryBB || EntryBB->getKnownExecutionCount() == 0)
+ return true;
+ Queue.push(EntryBB->getLayoutIndex());
+ Visited.insert(EntryBB->getLayoutIndex());
+ SumReachableBBEC += EntryBB->getKnownExecutionCount();
+ return true;
+ });
+
+ const FunctionLayout &Layout = Function->getLayout();
+
+ while (!Queue.empty()) {
+ const unsigned BBIndex = Queue.front();
+ const BinaryBasicBlock *BB = Layout.getBlock(BBIndex);
+ Queue.pop();
+ for (const auto &Tuple : llvm::zip(BB->successors(), BB->branch_info())) {
+ const auto *Succ = std::get<0>(Tuple);
+ const auto &BI = std::get<1>(Tuple);
+ const uint64_t Count = BI.Count;
+ if (Count == BinaryBasicBlock::COUNT_NO_PROFILE || Count == 0 ||
+ !Visited.insert(Succ->getLayoutIndex()).second)
+ continue;
+ SumReachableBBEC += Succ->getKnownExecutionCount();
+ Queue.push(Succ->getLayoutIndex());
+ }
+ }
+
+ const size_t NumReachableBBs = Visited.size();
+
+ const size_t NumPosECBBsUnreachableFromEntry =
+ NumPosECBBs - NumReachableBBs;
+ const size_t SumUnreachableBBEC = SumAllBBEC - SumReachableBBEC;
+ const double FractionECUnreachable =
+ (double)SumUnreachableBBEC / SumAllBBEC;
+
+ if (opts::Verbosity >= 2 && FractionECUnreachable >= 0.05) {
+ OS << "Non-trivial CFG discontinuity observed in function "
+ << Function->getPrintName() << "\n";
+ if (opts::Verbosity >= 3)
+ Function->dump();
+ }
+
+ NumUnreachables.push_back(NumPosECBBsUnreachableFromEntry);
+ SumECUnreachables.push_back(SumUnreachableBBEC);
+ FractionECUnreachables.push_back(FractionECUnreachable);
+ }
+
+ if (FractionECUnreachables.empty())
+ return;
+
+ llvm::sort(FractionECUnreachables);
+ const int Rank = int(FractionECUnreachables.size() *
+ opts::PercentileForProfileQualityCheck / 100);
+ OS << formatv("function CFG discontinuity {0:P}; ",
+ FractionECUnreachables[Rank]);
+ if (opts::Verbosity >= 1) {
+ OS << "\nabbreviations: EC = execution count, POS BBs = positive EC BBs\n"
+ << "distribution of NUM(unreachable POS BBs) per function\n";
+ llvm::sort(NumUnreachables);
+ printDistribution(OS, NumUnreachables);
+
+ OS << "distribution of SUM_EC(unreachable POS BBs) per function\n";
+ llvm::sort(SumECUnreachables);
+ printDistribution(OS, SumECUnreachables);
+
+ OS << "distribution of [(SUM_EC(unreachable POS BBs) / SUM_EC(all "
+ "POS BBs))] per function\n";
+ printDistribution(OS, FractionECUnreachables, /*Fraction=*/true);
+ }
+}
+
+void printCallGraphFlowConservationStats(
+ raw_ostream &OS, iterator_range<function_iterator> &Functions,
+ FlowInfo &TotalFlowMap) {
+ std::vector<double> CallGraphGaps;
+
+ for (const BinaryFunction *Function : Functions) {
+ if (Function->size() <= 1 || !Function->isSimple())
+ continue;
+
+ const uint64_t FunctionNum = Function->getFunctionNumber();
+ std::vector<uint64_t> &IncomingFlows =
+ TotalFlowMap.TotalIncomingFlows[FunctionNum];
+ std::vector<uint64_t> &OutgoingFlows =
+ TotalFlowMap.TotalOutgoingFlows[FunctionNum];
+ FunctionFlowMapTy &CallGraphIncomingFlows =
+ TotalFlowMap.CallGraphIncomingFlows;
+
+ // Only consider functions that are not a program entry.
+ if (CallGraphIncomingFlows.find(FunctionNum) !=
+ CallGraphIncomingFlows.end()) {
+ uint64_t EntryInflow = 0;
+ uint64_t EntryOutflow = 0;
+ uint32_t NumConsideredEntryBlocks = 0;
+
+ Function->forEachEntryPoint([&](uint64_t Offset, const MCSymbol *Label) {
+ const BinaryBasicBlock *EntryBB =
+ Function->getBasicBlockAtOffset(Offset);
+ if (!EntryBB || EntryBB->succ_size() == 0)
+ return true;
+ NumConsideredEntryBlocks++;
+ EntryInflow += IncomingFlows[EntryBB->getLayoutIndex()];
+ EntryOutflow += OutgoingFlows[EntryBB->getLayoutIndex()];
+ return true;
+ });
+
+ uint64_t NetEntryOutflow = 0;
+ if (EntryOutflow < EntryInflow) {
+ if (opts::Verbosity >= 2) {
+ // We expect entry blocks' CFG outflow >= inflow, i.e., it has a
+ // non-negative net outflow. If this is not the case, then raise a
+ // warning if requested.
+ OS << "BOLT WARNING: unexpected entry block CFG outflow < inflow "
+ "in function "
+ << Function->getPrintName() << "\n";
+ if (opts::Verbosity >= 3)
+ Function->dump();
+ }
+ } else {
+ NetEntryOutflow = EntryOutflow - EntryInflow;
+ }
+ if (NumConsideredEntryBlocks > 0) {
+ const uint64_t CallGraphInflow =
+ TotalFlowMap.CallGraphIncomingFlows[Function->getFunctionNumber()];
+ const uint64_t Min = std::min(NetEntryOutflow, CallGraphInflow);
+ const uint64_t Max = std::max(NetEntryOutflow, CallGraphInflow);
+ const double CallGraphGap = 1 - (double)Min / Max;
+
+ if (opts::Verbosity >= 2 && CallGraphGap >= 0.5) {
+ OS << "Nontrivial call graph gap of size "
+ << formatv("{0:P}", CallGraphGap) << " observed in function "
+ << Function->getPrintName() << "\n";
+ if (opts::Verbosity >= 3)
+ Function->dump();
+ }
+
+ CallGraphGaps.push_back(CallGraphGap);
+ }
+ }
+ }
+
+ if (CallGraphGaps.empty())
+ return;
+
+ llvm::sort(CallGraphGaps);
+ const int Rank =
+ int(CallGraphGaps.size() * opts::PercentileForProfileQualityCheck / 100);
+ OS << formatv("call graph flow conservation gap {0:P}; ",
+ CallGraphGaps[Rank]);
+ if (opts::Verbosity >= 1) {
+ OS << "\ndistribution of function entry flow conservation gaps\n";
+ printDistribution(OS, CallGraphGaps, /*Fraction=*/true);
+ }
+}
+
+void printCFGFlowConservationStats(raw_ostream &OS,
+ iterator_range<function_iterator> &Functions,
+ FlowInfo &TotalFlowMap) {
+ std::vector<double> CFGGapsWeightedAvg;
+ std::vector<double> CFGGapsWorst;
+ std::vector<uint64_t> CFGGapsWorstAbs;
+
+ for (const BinaryFunction *Function : Functions) {
+ if (Function->size() <= 1 || !Function->isSimple())
+ continue;
+
+ const uint64_t FunctionNum = Function->getFunctionNumber();
+ std::vector<uint64_t> &MaxCountMaps =
+ TotalFlowMap.TotalMaxCountMaps[FunctionNum];
+ std::vector<uint64_t> &MinCountMaps =
+ TotalFlowMap.TotalMinCountMaps[FunctionNum];
+ double WeightedGapSum = 0.0;
+ double WeightSum = 0.0;
+ double WorstGap = 0.0;
+ uint64_t WorstGapAbs = 0;
+ BinaryBasicBlock *BBWorstGap = nullptr;
+ BinaryBasicBlock *BBWorstGapAbs = nullptr;
+ for (BinaryBasicBlock &BB : *Function) {
+ // We don't consider function entry or exit blocks for CFG flow
+ // conservation
+ if (BB.isEntryPoint() || BB.succ_size() == 0)
+ continue;
+
+ const uint64_t Max = MaxCountMaps[BB.getLayoutIndex()];
+ const uint64_t Min = MinCountMaps[BB.getLayoutIndex()];
+ const double Gap = 1 - (double)Min / Max;
+ double Weight = BB.getKnownExecutionCount() * BB.getNumNonPseudos();
+ if (Weight == 0)
+ continue;
+ // We use log to prevent the stats from being dominated by extremely hot
+ // blocks
+ Weight = log(Weight);
+ WeightedGapSum += Gap * Weight;
+ WeightSum += Weight;
+ if (BB.getKnownExecutionCount() > 500 && Gap > WorstGap) {
----------------
aaupov wrote:
Please avoid magic constants (500), define a const variable instead.
https://github.com/llvm/llvm-project/pull/127954
More information about the llvm-commits
mailing list