[llvm] [PGOVerify] Add end-to-end IPGOVerifier infrastructure, validations, and function filtering (PR #196295)
via llvm-commits
llvm-commits at lists.llvm.org
Thu May 7 05:15:59 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Alok Kumar Sharma (alokkrsharma)
<details>
<summary>Changes</summary>
This PR combines 5 commits that introduce and expand IPGOVerifier so profile integrity can be checked after key optimization/instrumentation passes.
What this PR does:
- Adds the core IPGOVerifier infrastructure and hooks pass-manager callbacks so verification can run after passes.
- Adds block-frequency validation for PGOUse, including mismatch diagnostics.
- Adds function entry-count validation against caller-sum for PGOUse.
- Adds PGOGen instrumentation validation support.
- Adds function filtering support via verify-ipgo-funcs so verification can be scoped to selected functions.
Why:
- Makes profile correctness checks more systematic and easier to run after transformations.
- Improves diagnostic quality for both CFG-sensitive and count-consistency issues.
- Enables targeted debugging workflows by filtering verification to a subset of functions.
Notes:
- Diagnostics are available in normal verification mode and can be expanded in assert/debug workflows.
- Tests were updated to cover non-debug verification output paths and filtering behavior.
---
Patch is 132.35 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/196295.diff
41 Files Affected:
- (modified) llvm/include/llvm/Passes/StandardInstrumentations.h (+2)
- (added) llvm/include/llvm/Transforms/IPO/PGOVerify.h (+137)
- (modified) llvm/lib/Passes/StandardInstrumentations.cpp (+1)
- (modified) llvm/lib/Transforms/IPO/CMakeLists.txt (+1)
- (added) llvm/lib/Transforms/IPO/PGOVerify.cpp (+644)
- (added) llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-ipsccp.proftext (+20)
- (added) llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-merge-function.proftext (+25)
- (added) llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-edge-cases.proftext (+60)
- (added) llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-flow-conservation.proftext (+55)
- (added) llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-mother-block-frequency.proftext (+57)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-all-unknown-incoming-paths.ll (+43)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-block-edge-cases.ll (+205)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-block-flow-conservation.ll (+56)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-block-frequency-mismatch.ll (+135)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-cache-invalidation.ll (+17)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-cfg-transform-no-false-positives.ll (+88)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-cs-instr-summary-overflow-skip.ll (+52)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-caller-sum.ll (+112)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-internal.ll (+52)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-structured.ll (+51)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll (+59)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-externally-skip.ll (+26)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll (+59)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll (+24)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-ipsccp.ll (+61)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-large-entry-overflow-skip.ll (+82)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-loop-rotate.ll (+306)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll (+63)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-mother-proftext-flow.ll (+135)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-recursive-entry-mismatch.ll (+42)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-sample-summary-no-overflow-skip.ll (+58)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-available-externally.ll (+45)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-globally-disabled.ll (+52)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll (+14)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-strict-profile-counts.ll (+34)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-unknown-incoming-paths.ll (+52)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-unroll-cfg-change.ll (+75)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-use-pass-suppression.ll (+27)
- (added) llvm/test/Transforms/PGOVerifier/verify-ipgo-zero-func-count.ll (+165)
- (modified) llvm/unittests/Transforms/IPO/CMakeLists.txt (+1)
- (added) llvm/unittests/Transforms/IPO/PGOVerifyTest.cpp (+155)
``````````diff
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 4ee5ab2554868..5b8b2beff8647 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -29,6 +29,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/TimeProfiler.h"
+#include "llvm/Transforms/IPO/PGOVerify.h"
#include "llvm/Transforms/IPO/SampleProfileProbe.h"
#include <string>
@@ -612,6 +613,7 @@ class StandardInstrumentations {
IRChangedTester ChangeTester;
VerifyInstrumentation Verify;
DroppedVariableStatsIR DroppedStatsIR;
+ IPGOVerifier IPGOVerification;
bool VerifyEach;
diff --git a/llvm/include/llvm/Transforms/IPO/PGOVerify.h b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
new file mode 100644
index 0000000000000..0e5bdf1c148ad
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
@@ -0,0 +1,137 @@
+//===- Transforms/IPO/PGOVerify.h ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file provides the pass-instrumentation registration hook for
+/// `-verify-ipgo` diagnostics.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_PGOVERIFY_H
+#define LLVM_TRANSFORMS_IPO_PGOVERIFY_H
+
+#include "llvm/ADT/Any.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/LazyCallGraph.h"
+#include "llvm/IR/PassInstrumentation.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+class BasicBlock;
+class BlockFrequencyInfo;
+class Function;
+class Loop;
+class Module;
+class PassInstrumentationCallbacks;
+
+/// Registers `-verify-ipgo` diagnostics with pass instrumentation.
+class IPGOVerifier {
+public:
+ /// Per-block frequency state used by PGOVerifier flow checks.
+ struct BlockFreqInfo {
+ unsigned numUnknownIn = 0;
+ unsigned numUnknownOut = 0;
+ uint64_t sumIn = 0;
+ uint64_t sumOut = 0;
+ };
+
+ /// Frequency cache for all basic blocks in a function.
+ using AllBlockFreqInfo = MapVector<const BasicBlock *, BlockFreqInfo>;
+
+ /// Register post-pass callback hooks used by `-verify-ipgo` diagnostics.
+ ///
+ /// \param PIC Pass instrumentation callback registry.
+ LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC);
+
+ /// Dispatch post-pass handling by IR unit type.
+ ///
+ /// \param PassID Name of the pass that completed.
+ /// \param IR IR unit received from pass instrumentation callbacks.
+ LLVM_ABI void runAfterPass(StringRef PassID, Any IR);
+
+ /// Compute/infer block frequency state for flow-conservation checks.
+ LLVM_ABI void computeBlockFrequencies(const Function *F,
+ const BlockFrequencyInfo &BFI);
+
+ /// Retrieve cached per-block frequency information for a function.
+ ///
+ /// \note The cache is keyed by function pointer and is invalidated after
+ /// pass callbacks when IR may have changed.
+ ///
+ /// \return A pointer to cached frequency data for \p F, or `nullptr` when
+ /// no cache entry exists.
+ LLVM_ABI const AllBlockFreqInfo *
+ getCachedBlockFreqInfo(const Function *F) const;
+
+private:
+ /// Invalidate cached block-frequency entries for changed IR scopes.
+ void invalidateFunctionFrequencyCache(Any IR);
+
+ /// Return true if a function is eligible for verification.
+ ///
+ /// Applies verifier-local exclusions and optional command-line filtering.
+ bool shouldVerifyFunction(const Function *F) const;
+
+ /// Handle module callbacks by delegating each function to function handler.
+ void runAfterPass(const Module *M);
+
+ /// Per-function callback handler.
+ void runAfterPass(Function *F);
+
+ /// Handle SCC callbacks by delegating each function to function handler.
+ void runAfterPass(const LazyCallGraph::SCC *C);
+
+ /// Handle loop callbacks by delegating to containing function handler.
+ void runAfterPass(const Loop *L);
+
+ /// Check whether function-local profile counts may overflow 32-bit ranges.
+ ///
+ /// This guards strict flow-conservation checks that rely on bounded profile
+ /// counts derived from entry and block-level profile metadata.
+ ///
+ /// \return `true` if a possible overflow is detected for \p F, otherwise
+ /// `false`.
+ bool hasFunctionLocalCountOverflow(const Function *F, const llvm::BlockFrequencyInfo&) const;
+
+ /// Check whether a module carries an instrumentation-profile use summary.
+ ///
+ /// The verifier uses this as a signal that summary-based profile limits are
+ /// available to conservatively reason about local-count overflow.
+ ///
+ /// \return `true` if \p M has an instrumentation profile summary, otherwise
+ /// `false`.
+ bool hasInstrProfUseSummary(const Module *M) const;
+
+ /// Validate block-level flow conservation for known incoming/outgoing sums.
+ ///
+ /// For basic blocks whose incoming and outgoing frequency contributions are
+ /// fully known, this checks whether incoming sum equals outgoing sum.
+ /// Diagnostics are emitted in debug mode for mismatches or unknown states.
+ void validateBlockFrequencies(const Function *F);
+
+ /// Validate function entry count against summed direct-caller profile counts.
+ ///
+ /// This check runs only when the function has an entry count and all direct
+ /// callsites to the function have extractable profile totals.
+ void validateEntryCountAgainstCallerSum(const Function *F);
+
+ /// Validate instrumentation-generation phase invariants.
+ ///
+ /// Checks for gen-phase violations such as:
+ /// - InstrProf intrinsic names matching their containing function
+ /// - Counter global loads from the correct function
+ void verifyGenPhase(const Function *F);
+
+ /// Per-instance cache of inferred block-frequency data keyed by function.
+ DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
+};
+
+} // end namespace llvm
+#endif // LLVM_TRANSFORMS_IPO_PGOVERIFY_H
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 19e72a8612c4a..7f8fb09df8a32 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -2515,6 +2515,7 @@ void StandardInstrumentations::registerCallbacks(
OptPassGate.registerCallbacks(PIC);
PrintChangedIR.registerCallbacks(PIC);
PseudoProbeVerification.registerCallbacks(PIC);
+ IPGOVerification.registerCallbacks(PIC);
if (VerifyEach)
Verify.registerCallbacks(PIC, MAM);
PrintChangedDiff.registerCallbacks(PIC);
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index 1c4ee0336d4db..125fbcdd0da87 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -35,6 +35,7 @@ add_llvm_component_library(LLVMipo
ModuleInliner.cpp
OpenMPOpt.cpp
PartialInlining.cpp
+ PGOVerify.cpp
SampleContextTracker.cpp
SampleProfile.cpp
SampleProfileMatcher.cpp
diff --git a/llvm/lib/Transforms/IPO/PGOVerify.cpp b/llvm/lib/Transforms/IPO/PGOVerify.cpp
new file mode 100644
index 0000000000000..6dedf0093cf21
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/PGOVerify.cpp
@@ -0,0 +1,644 @@
+//===- PGOVerify.cpp - PGO Verification ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// IPGOVerifier currently provides registration-only diagnostics for
+// pass-instrumentation tracing under `-verify-ipgo`.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/PGOVerify.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfDataUtils.h"
+#include "llvm/IR/ProfileSummary.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include <limits>
+#include <numeric>
+#include <string>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "verify-ipgo"
+
+static cl::opt<bool> VerifyIPGOPrintDiagnostics(
+ "verify-ipgo-print-diagnostics", cl::init(true), cl::Hidden,
+ cl::desc("Print verify-ipgo diagnostics to stderr"));
+
+static cl::opt<bool>
+ VerifyIPGO("verify-ipgo", cl::init(false), cl::Hidden,
+ cl::desc("Enable Instrumented PGO verification"));
+
+static cl::list<std::string>
+ VerifyIPGOFuncList("verify-ipgo-funcs", cl::Hidden,
+ cl::desc("Comma-separated list of functions to verify"));
+
+/// Emit PGO verification diagnostics with structured formatting.
+///
+/// \param F Function being verified.
+/// \param RemarkName Diagnostic remark identifier.
+/// \param Msg Error/diagnostic message.
+static void emitPGOVerifyDiagnostic(const Function *F, StringRef RemarkName,
+ const Twine &Msg) {
+ std::string MsgText = Msg.str();
+ if (VerifyIPGOPrintDiagnostics)
+ errs() << "PGOVerify[" << RemarkName << "] " << F->getName() << ": "
+ << MsgText << "\n";
+ LLVM_DEBUG(dbgs() << "PGOVerify[" << RemarkName << "] " << F->getName()
+ << ": " << MsgText << "\n");
+}
+
+bool IPGOVerifier::shouldVerifyFunction(const Function *F) const {
+ if (!F || F->isDeclaration())
+ return false;
+
+ if (F->hasAvailableExternallyLinkage())
+ return false;
+
+ // Cache command-line function filters.
+ static const DenseSet<StringRef> FuncFilter = [] {
+ DenseSet<StringRef> S;
+ for (const auto &Func : VerifyIPGOFuncList)
+ S.insert(Func);
+ return S;
+ }();
+
+ return FuncFilter.empty() || FuncFilter.count(F->getName());
+}
+
+/// Register post-pass diagnostic callbacks for `-verify-ipgo`.
+///
+/// \param PIC Pass instrumentation callback registry.
+void IPGOVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
+ if (!VerifyIPGO)
+ return;
+
+ PIC.registerAfterPassCallback(
+ [this](StringRef PassName, Any IR, const PreservedAnalyses &PA) {
+ bool IsChanged = !PA.areAllPreserved();
+
+ StringRef Skipped = IsChanged ? "" : " (Skipped)";
+ if (VerifyIPGOPrintDiagnostics)
+ errs() << "*** IPGO Verification After " << PassName << Skipped
+ << " ***\n";
+ LLVM_DEBUG(dbgs() << "\n*** IPGO Verification After " << PassName
+ << Skipped << " ***\n");
+ if (!IsChanged) {
+ // Pass made no IR changes; skip verification.
+ return;
+ }
+
+ runAfterPass(PassName, IR);
+ });
+}
+
+/// Dispatch post-pass handling for supported IR unit kinds.
+///
+/// \param PassID Name of the pass that completed.
+/// \param IR IR unit received from pass instrumentation callbacks.
+void IPGOVerifier::runAfterPass(StringRef PassID, Any IR) {
+ (void)PassID;
+
+ // Drop cached per-function state for the IR unit that just changed before
+ // rebuilding or rechecking any derived block-frequency information.
+ invalidateFunctionFrequencyCache(IR);
+
+ if (const auto *M = any_cast<const Module *>(&IR))
+ runAfterPass(*M);
+ else if (const auto *F = any_cast<const Function *>(&IR)) {
+ // The verifier does not mutate IR, but the handler API is function-based,
+ // so adapt the callback payload here.
+ auto *NonConstF = const_cast<Function *>(*F);
+ runAfterPass(NonConstF);
+ } else if (const auto *C = any_cast<const LazyCallGraph::SCC *>(&IR))
+ runAfterPass(*C);
+ else if (const auto *L = any_cast<const Loop *>(&IR))
+ runAfterPass(*L);
+ else {
+ return;
+ }
+}
+
+void IPGOVerifier::invalidateFunctionFrequencyCache(Any IR) {
+ if (const auto *M = any_cast<const Module *>(&IR)) {
+ (void)M;
+ // Module passes can invalidate frequency state for any contained function.
+ FunctionBlockFreqInfoCache.clear();
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: module\n");
+ return;
+ }
+
+ if (const auto *F = any_cast<const Function *>(&IR)) {
+ FunctionBlockFreqInfoCache.erase(*F);
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: function\n");
+ return;
+ }
+
+ if (const auto *C = any_cast<const LazyCallGraph::SCC *>(&IR)) {
+ for (const LazyCallGraph::Node &N : **C)
+ FunctionBlockFreqInfoCache.erase(&N.getFunction());
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: scc\n");
+ return;
+ }
+
+ if (const auto *L = any_cast<const Loop *>(&IR)) {
+ FunctionBlockFreqInfoCache.erase((*L)->getHeader()->getParent());
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: loop\n");
+ return;
+ }
+
+ FunctionBlockFreqInfoCache.clear();
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: unknown\n");
+}
+
+/// Delegate module callback handling to the function handler.
+///
+/// \param M Module callback payload.
+void IPGOVerifier::runAfterPass(const Module *M) {
+ // Run Use-phase checks only when an InstrProf use summary is present.
+ if (M->getProfileSummary(/*IsCS=*/true))
+ return;
+ if (!hasInstrProfUseSummary(M)) {
+ for (const Function &F : *M) {
+ if (F.isDeclaration())
+ continue;
+ verifyGenPhase(&F);
+ }
+ return;
+ }
+ // First build frequency cache for all non-declaration functions so caller
+ // information is available regardless of function order in the module.
+ for (const Function &F : *M) {
+ if (F.isDeclaration())
+ continue;
+ // use BFI's non-synthetic per-block profile
+ // counts as the primary overflow signal.
+ auto *NonConstF = const_cast<Function *>(&F);
+ DominatorTree DT(*NonConstF);
+ LoopInfo LI(DT);
+ BranchProbabilityInfo BPI(*NonConstF, LI, nullptr, &DT, nullptr);
+ BlockFrequencyInfo BFI(*NonConstF, BPI, LI);
+
+ computeBlockFrequencies(&F, BFI);
+ }
+
+ // Then run validations using the populated cache.
+ for (const Function &F : *M) {
+ if (!shouldVerifyFunction(&F))
+ continue;
+ validateBlockFrequencies(&F);
+ validateEntryCountAgainstCallerSum(&F);
+ }
+}
+
+/// Per-function post-pass handler.
+///
+/// \param F Function callback payload.
+void IPGOVerifier::runAfterPass(Function *F) {
+ if (!F || F->isDeclaration() || !F->getParent())
+ return;
+
+ if (!shouldVerifyFunction(F))
+ return;
+
+ // Run Use-phase checks only when an InstrProf use summary is present.
+ if (F->getParent()->getProfileSummary(/*IsCS=*/true))
+ return;
+ if (!hasInstrProfUseSummary(F->getParent())) {
+ // Run Gen-phase checks (no dependencies on other passes).
+ verifyGenPhase(F);
+ return;
+ }
+
+ // Rebuild the minimal local analysis stack here so verification can query
+ // non-synthetic block profile counts after each pass callback.
+ DominatorTree DT(*F);
+ LoopInfo LI(DT);
+ BranchProbabilityInfo BPI(*F, LI, nullptr, &DT, nullptr);
+ BlockFrequencyInfo BFI(*F, BPI, LI);
+
+ computeBlockFrequencies(F, BFI);
+ validateBlockFrequencies(F);
+ validateEntryCountAgainstCallerSum(F);
+}
+
+/// Delegate SCC callback handling to the function handler.
+///
+/// \param C SCC callback payload.
+void IPGOVerifier::runAfterPass(const LazyCallGraph::SCC *C) {
+ for (const LazyCallGraph::Node &N : *C)
+ runAfterPass(&N.getFunction());
+}
+
+/// Delegate loop callback handling to the containing function handler.
+///
+/// \param L Loop callback payload.
+void IPGOVerifier::runAfterPass(const Loop *L) {
+ runAfterPass(L->getHeader()->getParent());
+}
+
+/// Compute and cache per-block flow state for verifier checks.
+///
+/// This seeds block-local incoming and outgoing totals from profile metadata,
+/// then iteratively propagates any facts that become forced by CFG structure
+/// until no additional block flow state can be resolved.
+///
+///
+/// \param F Function whose basic blocks are being analyzed.
+/// \param BFI BlockFrequencyInfo used to query non-synthetic profile counts
+/// and to detect cases where strict verification would be unsafe.
+void IPGOVerifier::computeBlockFrequencies(const Function *F,
+ const BlockFrequencyInfo &BFI) {
+ // Skip strict flow checks when local profile counts can overflow uint32.
+ if (hasFunctionLocalCountOverflow(F, BFI)) {
+ FunctionBlockFreqInfoCache[F] = AllBlockFreqInfo();
+ return;
+ }
+
+ AllBlockFreqInfo AllFreqInfo;
+
+ for (const BasicBlock &BB : *F) {
+ // Start with all predecessor and successor contributions unknown, then
+ // refine each block as profile metadata or structural rules provide facts.
+ AllFreqInfo[&BB].numUnknownIn = llvm::pred_size(&BB);
+ AllFreqInfo[&BB].numUnknownOut = llvm::succ_size(&BB);
+ AllFreqInfo[&BB].sumIn = 0;
+ AllFreqInfo[&BB].sumOut = 0;
+ }
+
+ // Model the function entry as an external incoming edge so the entry count
+ // can seed flow-conservation reasoning like any other known predecessor.
+ AllFreqInfo[&F->getEntryBlock()].numUnknownIn = 1;
+
+ if (auto Count = F->getEntryCount()) {
+ AllFreqInfo[&F->getEntryBlock()].sumIn = Count->getCount();
+ AllFreqInfo[&F->getEntryBlock()].numUnknownIn = 0;
+ if (Count->getCount() == 0) {
+ // A zero entry count forces every reachable block contribution to zero,
+ // which avoids leaving unknown edges behind in dead-profile functions.
+ for (const BasicBlock &BB : *F) {
+ AllFreqInfo[&BB].numUnknownIn = 0;
+ AllFreqInfo[&BB].sumIn = 0;
+ AllFreqInfo[&BB].numUnknownOut = 0;
+ AllFreqInfo[&BB].sumOut = 0;
+ }
+ } else {
+ const Instruction *Term = F->getEntryBlock().getTerminator();
+ if (Term && (Term->getNumSuccessors() == 0)) {
+ AllFreqInfo[&F->getEntryBlock()].sumOut = Count->getCount();
+ AllFreqInfo[&F->getEntryBlock()].numUnknownOut = 0;
+ }
+ }
+ }
+
+ for (const BasicBlock &BB : *F) {
+ SmallVector<uint64_t> Weights;
+ const Instruction *Term = BB.getTerminator();
+ if (!Term)
+ continue;
+
+ if (isa<ReturnInst>(Term) && AllFreqInfo[&BB].numUnknownIn == 0) {
+ AllFreqInfo[&BB].sumOut = AllFreqInfo[&BB].sumIn;
+ AllFreqInfo[&BB].numUnknownOut = 0;
+ continue;
+ }
+
+ if (MDNode *Prof = Term->getMetadata(LLVMContext::MD_prof)) {
+ if (Prof->getNumOperands() > 1) {
+ for (unsigned I = 1; I < Prof->getNumOperands(); ++I) {
+ auto *CI = mdconst::dyn_extract<ConstantInt>(Prof->getOperand(I));
+ if (!CI) {
+ // Ignore malformed weight metadata and leave the block unresolved.
+ Weights.clear();
+ break;
+ }
+ Weights.push_back(CI->getZExtValue());
+ }
+ }
+ }
+
+ if (Weights.empty())
+ continue;
+
+ if (Weights.size() != Term->getNumSuccessors())
+ continue;
+
+ // Explicit successor weights fully determine the outgoing total for this
+ // terminator and contribute known incoming counts to each successor.
+ for (unsigned I = 0; I < Term->getNumSuccessors(); ++I) {
+ if (AllFreqInfo[Term->getSuccessor(I)].numUnknownIn > 0)
+ AllFreqInfo[Term->getSuccessor(I)].numUnknownIn--;
+ AllFreqInfo[Term->getSuccessor(I)].sumIn += Weights[I];
+ }
+ AllFreqInfo[&BB].numUnknownOut = 0;
+ AllFreqInfo[&BB].sumOut =
+ std::accumulate(Weights.begin(), Weights.end(), uint64_t(0));
+ }
+
+ bool Changed = false;
+ do {
+ Changed = false;
+ for (const BasicBlock &BB : *F) {
+ const Instruction *Term = BB.getTerminator();
+ if (!Term)
+ continue;
+
+ // Once a block is known to receive zero flow, every still-unknown exit
+ // edge from that block can also be fixed to zero.
+ if (AllFreqIn...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/196295
More information about the llvm-commits
mailing list