[llvm] [PGOVerify] Add end-to-end IPGOVerifier infrastructure, validations, and function filtering (PR #196295)
Alok Kumar Sharma via llvm-commits
llvm-commits at lists.llvm.org
Thu May 7 05:15:04 PDT 2026
https://github.com/alokkrsharma created https://github.com/llvm/llvm-project/pull/196295
This PR combines 5 commits that introduce and expand IPGOVerifier so profile integrity can be checked after key optimization/instrumentation passes.
What this PR does:
- Adds the core IPGOVerifier infrastructure and hooks pass-manager callbacks so verification can run after passes.
- Adds block-frequency validation for PGOUse, including mismatch diagnostics.
- Adds function entry-count validation against caller-sum for PGOUse.
- Adds PGOGen instrumentation validation support.
- Adds function filtering support via verify-ipgo-funcs so verification can be scoped to selected functions.
Why:
- Makes profile correctness checks more systematic and easier to run after transformations.
- Improves diagnostic quality for both CFG-sensitive and count-consistency issues.
- Enables targeted debugging workflows by filtering verification to a subset of functions.
Notes:
- Diagnostics are available in normal verification mode and can be expanded in assert/debug workflows.
- Tests were updated to cover non-debug verification output paths and filtering behavior.
>From 40bb2b400d37ab7ad170d2b97c4620b9625c5cfc Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Tue, 5 May 2026 11:04:00 +0530
Subject: [PATCH 1/5] [PGOVerify] [1/5] Add IPGOVerifier infrastructure for
post-pass validation
Adds PassInstrumentation callbacks to enable PGO verification after IR-changing
passes. Registers IPGOVerifier in the standard instrumentations pipeline to
detect profile data inconsistencies.
---
.../llvm/Passes/StandardInstrumentations.h | 2 +
llvm/include/llvm/Transforms/IPO/PGOVerify.h | 58 +++++++++
llvm/lib/Passes/StandardInstrumentations.cpp | 1 +
llvm/lib/Transforms/IPO/CMakeLists.txt | 1 +
llvm/lib/Transforms/IPO/PGOVerify.cpp | 114 ++++++++++++++++++
.../verify-ipgo-skipped-diagnostics.ll | 13 ++
6 files changed, 189 insertions(+)
create mode 100644 llvm/include/llvm/Transforms/IPO/PGOVerify.h
create mode 100644 llvm/lib/Transforms/IPO/PGOVerify.cpp
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 4ee5ab2554868..5b8b2beff8647 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -29,6 +29,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/TimeProfiler.h"
+#include "llvm/Transforms/IPO/PGOVerify.h"
#include "llvm/Transforms/IPO/SampleProfileProbe.h"
#include <string>
@@ -612,6 +613,7 @@ class StandardInstrumentations {
IRChangedTester ChangeTester;
VerifyInstrumentation Verify;
DroppedVariableStatsIR DroppedStatsIR;
+ IPGOVerifier IPGOVerification;
bool VerifyEach;
diff --git a/llvm/include/llvm/Transforms/IPO/PGOVerify.h b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
new file mode 100644
index 0000000000000..45349e84c434d
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
@@ -0,0 +1,58 @@
+//===- Transforms/IPO/PGOVerify.h ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file provides the pass-instrumentation registration hook for
+/// `-verify-ipgo` diagnostics.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_PGOVERIFY_H
+#define LLVM_TRANSFORMS_IPO_PGOVERIFY_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/LazyCallGraph.h"
+#include "llvm/IR/PassInstrumentation.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+class Function;
+class Loop;
+class Module;
+class PassInstrumentationCallbacks;
+
+/// Registers `-verify-ipgo` diagnostics with pass instrumentation.
+class IPGOVerifier {
+public:
+ /// Register post-pass callback hooks used by `-verify-ipgo` diagnostics.
+ ///
+ /// \param PIC Pass instrumentation callback registry.
+ LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC);
+
+ /// Dispatch post-pass handling by IR unit type.
+ ///
+ /// \param PassID Name of the pass that completed.
+ /// \param IR IR unit received from pass instrumentation callbacks.
+ LLVM_ABI void runAfterPass(StringRef PassID, Any IR);
+
+private:
+ /// Handle module callbacks by delegating each function to function handler.
+ void runAfterPass(const Module *M);
+
+ /// Per-function callback handler.
+ void runAfterPass(const Function *F);
+
+ /// Handle SCC callbacks by delegating each function to function handler.
+ void runAfterPass(const LazyCallGraph::SCC *C);
+
+ /// Handle loop callbacks by delegating to containing function handler.
+ void runAfterPass(const Loop *L);
+};
+
+} // end namespace llvm
+#endif // LLVM_TRANSFORMS_IPO_PGOVERIFY_H
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 19e72a8612c4a..7f8fb09df8a32 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -2515,6 +2515,7 @@ void StandardInstrumentations::registerCallbacks(
OptPassGate.registerCallbacks(PIC);
PrintChangedIR.registerCallbacks(PIC);
PseudoProbeVerification.registerCallbacks(PIC);
+ IPGOVerification.registerCallbacks(PIC);
if (VerifyEach)
Verify.registerCallbacks(PIC, MAM);
PrintChangedDiff.registerCallbacks(PIC);
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index 1c4ee0336d4db..125fbcdd0da87 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -35,6 +35,7 @@ add_llvm_component_library(LLVMipo
ModuleInliner.cpp
OpenMPOpt.cpp
PartialInlining.cpp
+ PGOVerify.cpp
SampleContextTracker.cpp
SampleProfile.cpp
SampleProfileMatcher.cpp
diff --git a/llvm/lib/Transforms/IPO/PGOVerify.cpp b/llvm/lib/Transforms/IPO/PGOVerify.cpp
new file mode 100644
index 0000000000000..20c0672d3f196
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/PGOVerify.cpp
@@ -0,0 +1,114 @@
+//===- PGOVerify.cpp - PGO Verification ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// IPGOVerifier currently provides registration-only diagnostics for
+// pass-instrumentation tracing under `-verify-ipgo`.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/PGOVerify.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "verify-ipgo"
+
+static cl::opt<bool> VerifyIPGOPrintDiagnostics(
+ "verify-ipgo-print-diagnostics", cl::init(true), cl::Hidden,
+ cl::desc("Print verify-ipgo diagnostics to stderr"));
+
+static cl::opt<bool>
+ VerifyIPGO("verify-ipgo", cl::init(false), cl::Hidden,
+ cl::desc("Enable Instrumented PGO verification"));
+
+/// Register post-pass diagnostic callbacks for `-verify-ipgo`.
+///
+/// \param PIC Pass instrumentation callback registry.
+void IPGOVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
+ if (!VerifyIPGO)
+ return;
+
+ PIC.registerAfterPassCallback(
+ [this](StringRef PassName, Any IR, const PreservedAnalyses &PA) {
+ bool IsChanged = !PA.areAllPreserved();
+
+ StringRef Skipped = IsChanged ? "" : " (Skipped)";
+ if (VerifyIPGOPrintDiagnostics)
+ errs() << "*** IPGO Verification After " << PassName << Skipped
+ << " ***\n";
+ LLVM_DEBUG(dbgs() << "\n*** IPGO Verification After " << PassName
+ << Skipped << " ***\n");
+ if (!IsChanged) {
+ // Pass made no IR changes; skip verification.
+ return;
+ }
+
+ runAfterPass(PassName, IR);
+ });
+}
+
+/// Dispatch post-pass handling for supported IR unit kinds.
+///
+/// \param PassID Name of the pass that completed.
+/// \param IR IR unit received from pass instrumentation callbacks.
+void IPGOVerifier::runAfterPass(StringRef PassID, Any IR) {
+ (void)PassID;
+
+ if (const auto *M = any_cast<const Module *>(&IR))
+ runAfterPass(*M);
+ else if (const auto *F = any_cast<const Function *>(&IR)) {
+ // The verifier does not mutate IR, but the handler API is function-based,
+ // so adapt the callback payload here.
+ auto *NonConstF = const_cast<Function *>(*F);
+ runAfterPass(NonConstF);
+ } else if (const auto *C = any_cast<const LazyCallGraph::SCC *>(&IR))
+ runAfterPass(*C);
+ else if (const auto *L = any_cast<const Loop *>(&IR))
+ runAfterPass(*L);
+ else {
+ return;
+ }
+}
+
+/// Delegate module callback handling to the function handler.
+///
+/// \param M Module callback payload.
+void IPGOVerifier::runAfterPass(const Module *M) {
+ for (const Function &F : *M) {
+ if (F.isDeclaration())
+ continue;
+
+ runAfterPass(const_cast<Function *>(&F));
+ }
+}
+
+/// Per-function post-pass handler.
+///
+/// \param F Function callback payload.
+void IPGOVerifier::runAfterPass(const Function *F) {
+ if (!F || F->isDeclaration())
+ return;
+}
+
+/// Delegate SCC callback handling to the function handler.
+///
+/// \param C SCC callback payload.
+void IPGOVerifier::runAfterPass(const LazyCallGraph::SCC *C) {
+ for (const LazyCallGraph::Node &N : *C)
+ runAfterPass(&N.getFunction());
+}
+
+/// Delegate loop callback handling to the containing function handler.
+///
+/// \param L Loop callback payload.
+void IPGOVerifier::runAfterPass(const Loop *L) {
+ runAfterPass(L->getHeader()->getParent());
+}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll
new file mode 100644
index 0000000000000..ae15e3c36979e
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes='instcombine,instcombine' -verify-ipgo -disable-output %s 2>&1 | FileCheck %s
+;
+; Ensure verify-ipgo callback diagnostics include both changed and skipped forms.
+
+; CHECK: *** IPGO Verification After InstCombinePass ***
+; CHECK: *** IPGO Verification After InstCombinePass (Skipped) ***
+
+define i32 @f(i32 %x) {
+entry:
+ %a = add i32 %x, 0
+ ret i32 %a
+}
>From afc3f05db75c7c2e9ea4dd6e28fbeb8cfb4f4ffd Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Tue, 5 May 2026 11:05:13 +0530
Subject: [PATCH 2/5] [PGOVerify] [2/5] Implement block frequency validation in
IPGOVerifier
Adds flow conservation validation to detect incoming/outgoing frequency
mismatches in basic blocks. Includes comprehensive tests for loops, CFG
transforms, and edge cases.
---
llvm/include/llvm/Transforms/IPO/PGOVerify.h | 62 +++-
llvm/lib/Transforms/IPO/PGOVerify.cpp | 309 +++++++++++++++++-
.../verify-ipgo-block-edge-cases.proftext | 60 ++++
...rify-ipgo-block-flow-conservation.proftext | 55 ++++
...erify-ipgo-mother-block-frequency.proftext | 57 ++++
.../verify-ipgo-block-edge-cases.ll | 205 ++++++++++++
.../verify-ipgo-block-flow-conservation.ll | 56 ++++
.../verify-ipgo-block-frequency-mismatch.ll | 135 ++++++++
.../verify-ipgo-cache-invalidation.ll | 17 +
...y-ipgo-cfg-transform-no-false-positives.ll | 88 +++++
...ify-ipgo-cs-instr-summary-overflow-skip.ll | 52 +++
.../verify-ipgo-gen-counter-load-mismatch.ll | 23 ++
.../verify-ipgo-large-entry-overflow-skip.ll | 82 +++++
.../PGOVerifier/verify-ipgo-loop-rotate.ll | 306 +++++++++++++++++
.../verify-ipgo-mother-proftext-flow.ll | 135 ++++++++
...fy-ipgo-sample-summary-no-overflow-skip.ll | 58 ++++
.../verify-ipgo-skip-available-externally.ll | 45 +++
.../verify-ipgo-skip-globally-disabled.ll | 52 +++
.../verify-ipgo-unroll-cfg-change.ll | 75 +++++
llvm/unittests/Transforms/IPO/CMakeLists.txt | 1 +
.../Transforms/IPO/PGOVerifyTest.cpp | 155 +++++++++
21 files changed, 2026 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-edge-cases.proftext
create mode 100644 llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-flow-conservation.proftext
create mode 100644 llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-mother-block-frequency.proftext
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-block-edge-cases.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-block-flow-conservation.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-block-frequency-mismatch.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-cache-invalidation.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-cfg-transform-no-false-positives.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-cs-instr-summary-overflow-skip.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-large-entry-overflow-skip.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-loop-rotate.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-mother-proftext-flow.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-sample-summary-no-overflow-skip.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-available-externally.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-globally-disabled.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-unroll-cfg-change.ll
create mode 100644 llvm/unittests/Transforms/IPO/PGOVerifyTest.cpp
diff --git a/llvm/include/llvm/Transforms/IPO/PGOVerify.h b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
index 45349e84c434d..aec4a1832bc46 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
@@ -15,12 +15,16 @@
#ifndef LLVM_TRANSFORMS_IPO_PGOVERIFY_H
#define LLVM_TRANSFORMS_IPO_PGOVERIFY_H
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/IR/PassInstrumentation.h"
#include "llvm/Support/Compiler.h"
namespace llvm {
+class BasicBlock;
+class BlockFrequencyInfo;
class Function;
class Loop;
class Module;
@@ -29,6 +33,17 @@ class PassInstrumentationCallbacks;
/// Registers `-verify-ipgo` diagnostics with pass instrumentation.
class IPGOVerifier {
public:
+ /// Per-block frequency state used by PGOVerifier flow checks.
+ struct BlockFreqInfo {
+ unsigned numUnknownIn = 0;
+ unsigned numUnknownOut = 0;
+ uint64_t sumIn = 0;
+ uint64_t sumOut = 0;
+ };
+
+ /// Frequency cache for all basic blocks in a function.
+ using AllBlockFreqInfo = MapVector<const BasicBlock *, BlockFreqInfo>;
+
/// Register post-pass callback hooks used by `-verify-ipgo` diagnostics.
///
/// \param PIC Pass instrumentation callback registry.
@@ -40,18 +55,63 @@ class IPGOVerifier {
/// \param IR IR unit received from pass instrumentation callbacks.
LLVM_ABI void runAfterPass(StringRef PassID, Any IR);
+ /// Compute/infer block frequency state for flow-conservation checks.
+ LLVM_ABI void computeBlockFrequencies(const Function *F,
+ const BlockFrequencyInfo &BFI);
+
+ /// Retrieve cached per-block frequency information for a function.
+ ///
+ /// \note The cache is keyed by function pointer and is invalidated after
+ /// pass callbacks when IR may have changed.
+ ///
+ /// \return A pointer to cached frequency data for \p F, or `nullptr` when
+ /// no cache entry exists.
+ LLVM_ABI const AllBlockFreqInfo *
+ getCachedBlockFreqInfo(const Function *F) const;
+
private:
+ /// Invalidate cached block-frequency entries for changed IR scopes.
+ void invalidateFunctionFrequencyCache(Any IR);
+
/// Handle module callbacks by delegating each function to function handler.
void runAfterPass(const Module *M);
/// Per-function callback handler.
- void runAfterPass(const Function *F);
+ void runAfterPass(Function *F);
/// Handle SCC callbacks by delegating each function to function handler.
void runAfterPass(const LazyCallGraph::SCC *C);
/// Handle loop callbacks by delegating to containing function handler.
void runAfterPass(const Loop *L);
+
+ /// Check whether function-local profile counts may overflow 32-bit ranges.
+ ///
+ /// This guards strict flow-conservation checks that rely on bounded profile
+ /// counts derived from entry and block-level profile metadata.
+ ///
+ /// \return `true` if a possible overflow is detected for \p F, otherwise
+ /// `false`.
+ bool hasFunctionLocalCountOverflow(const Function *F, const llvm::BlockFrequencyInfo&) const;
+
+ /// Check whether a module carries an instrumentation-profile use summary.
+ ///
+ /// The verifier uses this as a signal that summary-based profile limits are
+ /// available to conservatively reason about local-count overflow.
+ ///
+ /// \return `true` if \p M has an instrumentation profile summary, otherwise
+ /// `false`.
+ bool hasInstrProfUseSummary(const Module *M) const;
+
+ /// Validate block-level flow conservation for known incoming/outgoing sums.
+ ///
+ /// For basic blocks whose incoming and outgoing frequency contributions are
+ /// fully known, this checks whether incoming sum equals outgoing sum.
+ /// Diagnostics are emitted in debug mode for mismatches or unknown states.
+ void validateBlockFrequencies(const Function *F);
+
+ /// Per-instance cache of inferred block-frequency data keyed by function.
+ DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/IPO/PGOVerify.cpp b/llvm/lib/Transforms/IPO/PGOVerify.cpp
index 20c0672d3f196..692bc1e6a0bda 100644
--- a/llvm/lib/Transforms/IPO/PGOVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOVerify.cpp
@@ -12,10 +12,18 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/IPO/PGOVerify.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfDataUtils.h"
+#include "llvm/IR/ProfileSummary.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
+#include <limits>
+#include <numeric>
using namespace llvm;
@@ -62,6 +70,10 @@ void IPGOVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
void IPGOVerifier::runAfterPass(StringRef PassID, Any IR) {
(void)PassID;
+ // Drop cached per-function state for the IR unit that just changed before
+ // rebuilding or rechecking any derived block-frequency information.
+ invalidateFunctionFrequencyCache(IR);
+
if (const auto *M = any_cast<const Module *>(&IR))
runAfterPass(*M);
else if (const auto *F = any_cast<const Function *>(&IR)) {
@@ -78,10 +90,46 @@ void IPGOVerifier::runAfterPass(StringRef PassID, Any IR) {
}
}
+void IPGOVerifier::invalidateFunctionFrequencyCache(Any IR) {
+ if (const auto *M = any_cast<const Module *>(&IR)) {
+ (void)M;
+ // Module passes can invalidate frequency state for any contained function.
+ FunctionBlockFreqInfoCache.clear();
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: module\n");
+ return;
+ }
+
+ if (const auto *F = any_cast<const Function *>(&IR)) {
+ FunctionBlockFreqInfoCache.erase(*F);
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: function\n");
+ return;
+ }
+
+ if (const auto *C = any_cast<const LazyCallGraph::SCC *>(&IR)) {
+ for (const LazyCallGraph::Node &N : **C)
+ FunctionBlockFreqInfoCache.erase(&N.getFunction());
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: scc\n");
+ return;
+ }
+
+ if (const auto *L = any_cast<const Loop *>(&IR)) {
+ FunctionBlockFreqInfoCache.erase((*L)->getHeader()->getParent());
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: loop\n");
+ return;
+ }
+
+ FunctionBlockFreqInfoCache.clear();
+ LLVM_DEBUG(dbgs() << "PGOVerify cache invalidated: unknown\n");
+}
+
/// Delegate module callback handling to the function handler.
///
/// \param M Module callback payload.
void IPGOVerifier::runAfterPass(const Module *M) {
+ // Run Use-phase checks only when an InstrProf use summary is present.
+ if (!hasInstrProfUseSummary(M))
+ return;
+
for (const Function &F : *M) {
if (F.isDeclaration())
continue;
@@ -93,9 +141,23 @@ void IPGOVerifier::runAfterPass(const Module *M) {
/// Per-function post-pass handler.
///
/// \param F Function callback payload.
-void IPGOVerifier::runAfterPass(const Function *F) {
+void IPGOVerifier::runAfterPass(Function *F) {
if (!F || F->isDeclaration())
return;
+
+ // Run Use-phase checks only when an InstrProf use summary is present.
+ if (!hasInstrProfUseSummary(F->getParent()))
+ return;
+
+ // Rebuild the minimal local analysis stack here so verification can query
+ // non-synthetic block profile counts after each pass callback.
+ DominatorTree DT(*F);
+ LoopInfo LI(DT);
+ BranchProbabilityInfo BPI(*F, LI, nullptr, &DT, nullptr);
+ BlockFrequencyInfo BFI(*F, BPI, LI);
+
+ computeBlockFrequencies(F, BFI);
+ validateBlockFrequencies(F);
}
/// Delegate SCC callback handling to the function handler.
@@ -112,3 +174,248 @@ void IPGOVerifier::runAfterPass(const LazyCallGraph::SCC *C) {
void IPGOVerifier::runAfterPass(const Loop *L) {
runAfterPass(L->getHeader()->getParent());
}
+
+/// Compute and cache per-block flow state for verifier checks.
+///
+/// This seeds block-local incoming and outgoing totals from profile metadata,
+/// then iteratively propagates any facts that become forced by CFG structure
+/// until no additional block flow state can be resolved.
+///
+///
+/// \param F Function whose basic blocks are being analyzed.
+/// \param BFI BlockFrequencyInfo used to query non-synthetic profile counts
+/// and to detect cases where strict verification would be unsafe.
+void IPGOVerifier::computeBlockFrequencies(const Function *F,
+ const BlockFrequencyInfo &BFI) {
+ // Skip strict flow checks when local profile counts can overflow uint32.
+ if (hasFunctionLocalCountOverflow(F, BFI)) {
+ FunctionBlockFreqInfoCache[F] = AllBlockFreqInfo();
+ return;
+ }
+
+ AllBlockFreqInfo AllFreqInfo;
+
+ for (const BasicBlock &BB : *F) {
+ // Start with all predecessor and successor contributions unknown, then
+ // refine each block as profile metadata or structural rules provide facts.
+ AllFreqInfo[&BB].numUnknownIn = llvm::pred_size(&BB);
+ AllFreqInfo[&BB].numUnknownOut = llvm::succ_size(&BB);
+ AllFreqInfo[&BB].sumIn = 0;
+ AllFreqInfo[&BB].sumOut = 0;
+ }
+
+ // Model the function entry as an external incoming edge so the entry count
+ // can seed flow-conservation reasoning like any other known predecessor.
+ AllFreqInfo[&F->getEntryBlock()].numUnknownIn = 1;
+
+ if (auto Count = F->getEntryCount()) {
+ AllFreqInfo[&F->getEntryBlock()].sumIn = Count->getCount();
+ AllFreqInfo[&F->getEntryBlock()].numUnknownIn = 0;
+ if (Count->getCount() == 0) {
+ // A zero entry count forces every reachable block contribution to zero,
+ // which avoids leaving unknown edges behind in dead-profile functions.
+ for (const BasicBlock &BB : *F) {
+ AllFreqInfo[&BB].numUnknownIn = 0;
+ AllFreqInfo[&BB].sumIn = 0;
+ AllFreqInfo[&BB].numUnknownOut = 0;
+ AllFreqInfo[&BB].sumOut = 0;
+ }
+ } else {
+ const Instruction *Term = F->getEntryBlock().getTerminator();
+ if (Term && (Term->getNumSuccessors() == 0)) {
+ AllFreqInfo[&F->getEntryBlock()].sumOut = Count->getCount();
+ AllFreqInfo[&F->getEntryBlock()].numUnknownOut = 0;
+ }
+ }
+ }
+
+ for (const BasicBlock &BB : *F) {
+ SmallVector<uint64_t> Weights;
+ const Instruction *Term = BB.getTerminator();
+ if (!Term)
+ continue;
+
+ if (isa<ReturnInst>(Term) && AllFreqInfo[&BB].numUnknownIn == 0) {
+ AllFreqInfo[&BB].sumOut = AllFreqInfo[&BB].sumIn;
+ AllFreqInfo[&BB].numUnknownOut = 0;
+ continue;
+ }
+
+ if (MDNode *Prof = Term->getMetadata(LLVMContext::MD_prof)) {
+ if (Prof->getNumOperands() > 1) {
+ for (unsigned I = 1; I < Prof->getNumOperands(); ++I) {
+ auto *CI = mdconst::dyn_extract<ConstantInt>(Prof->getOperand(I));
+ if (!CI) {
+ // Ignore malformed weight metadata and leave the block unresolved.
+ Weights.clear();
+ break;
+ }
+ Weights.push_back(CI->getZExtValue());
+ }
+ }
+ }
+
+ if (Weights.empty())
+ continue;
+
+ if (Weights.size() != Term->getNumSuccessors())
+ continue;
+
+ // Explicit successor weights fully determine the outgoing total for this
+ // terminator and contribute known incoming counts to each successor.
+ for (unsigned I = 0; I < Term->getNumSuccessors(); ++I) {
+ if (AllFreqInfo[Term->getSuccessor(I)].numUnknownIn > 0)
+ AllFreqInfo[Term->getSuccessor(I)].numUnknownIn--;
+ AllFreqInfo[Term->getSuccessor(I)].sumIn += Weights[I];
+ }
+ AllFreqInfo[&BB].numUnknownOut = 0;
+ AllFreqInfo[&BB].sumOut =
+ std::accumulate(Weights.begin(), Weights.end(), uint64_t(0));
+ }
+
+ bool Changed = false;
+ do {
+ Changed = false;
+ for (const BasicBlock &BB : *F) {
+ const Instruction *Term = BB.getTerminator();
+ if (!Term)
+ continue;
+
+ // Once a block is known to receive zero flow, every still-unknown exit
+ // edge from that block can also be fixed to zero.
+ if (AllFreqInfo[&BB].numUnknownIn == 0 && AllFreqInfo[&BB].sumIn == 0 &&
+ AllFreqInfo[&BB].numUnknownOut > 0) {
+ for (unsigned I = 0; I < Term->getNumSuccessors(); ++I)
+ if (AllFreqInfo[Term->getSuccessor(I)].numUnknownIn > 0)
+ AllFreqInfo[Term->getSuccessor(I)].numUnknownIn--;
+ AllFreqInfo[&BB].numUnknownOut = 0;
+ AllFreqInfo[&BB].sumOut = 0;
+
+ Changed = true;
+ continue;
+ }
+
+ // A single unresolved successor on a single-successor terminator must
+ // carry the entire incoming flow for this block.
+ if (AllFreqInfo[&BB].numUnknownIn == 0 &&
+ AllFreqInfo[&BB].numUnknownOut == 1) {
+ if (Term->getNumSuccessors() > 1)
+ continue;
+
+ for (unsigned I = 0; I < Term->getNumSuccessors(); ++I) {
+ if (AllFreqInfo[Term->getSuccessor(I)].numUnknownIn > 0)
+ AllFreqInfo[Term->getSuccessor(I)].numUnknownIn--;
+ AllFreqInfo[Term->getSuccessor(I)].sumIn += AllFreqInfo[&BB].sumIn;
+ }
+ AllFreqInfo[&BB].numUnknownOut = 0;
+ AllFreqInfo[&BB].sumOut = AllFreqInfo[&BB].sumIn;
+
+ Changed = true;
+ }
+ }
+ } while (Changed);
+
+ FunctionBlockFreqInfoCache[F] = AllFreqInfo;
+}
+
+const IPGOVerifier::AllBlockFreqInfo *
+IPGOVerifier::getCachedBlockFreqInfo(const Function *F) const {
+ auto It = FunctionBlockFreqInfoCache.find(F);
+ if (It == FunctionBlockFreqInfoCache.end())
+ return nullptr;
+ return &It->second;
+}
+
+bool IPGOVerifier::hasFunctionLocalCountOverflow(const Function *F,
+ const BlockFrequencyInfo &BFI) const {
+ constexpr uint64_t UInt32Max = std::numeric_limits<uint32_t>::max();
+
+ if (auto EntryCount = F->getEntryCount();
+ EntryCount && EntryCount->getCount() > UInt32Max)
+ return true;
+
+
+ bool HasUnknownBlockCount = false;
+ for (const BasicBlock &BB : *F) {
+ if (&BB == &F->getEntryBlock())
+ continue;
+
+ // Only trust non-synthetic counts here; synthetic counts may already be
+ // inferred from the CFG and would circularly justify verifier results.
+ if (std::optional<uint64_t> Count =
+ BFI.getBlockProfileCount(&BB, /*AllowSynthetic=*/false);
+ Count && *Count > UInt32Max)
+ return true;
+ else if (!Count)
+ HasUnknownBlockCount = true;
+ }
+
+ // Conservative fallback when some per-block counts are unavailable.
+ if (HasUnknownBlockCount) {
+ if (const Module *M = F->getParent()) {
+ // The profile summary gives an upper bound when local block counts are
+ // missing, allowing the verifier to avoid strict checks near overflow.
+ Metadata *SummaryMD = M->getProfileSummary(/*IsCS=*/false);
+ if (SummaryMD) {
+ std::unique_ptr<ProfileSummary> PS(
+ ProfileSummary::getFromMD(SummaryMD));
+ if (PS && PS->getMaxInternalCount() > UInt32Max)
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+bool IPGOVerifier::hasInstrProfUseSummary(const Module *M) const {
+ if (!M)
+ return false;
+
+ Metadata *SummaryMD = M->getProfileSummary(/*IsCS=*/false);
+ if (!SummaryMD)
+ return false;
+
+ std::unique_ptr<ProfileSummary> PS(ProfileSummary::getFromMD(SummaryMD));
+ return PS && PS->getKind() == ProfileSummary::PSK_Instr;
+}
+
+/// Validate flow conservation where both sides are known.
+void IPGOVerifier::validateBlockFrequencies(const Function *F) {
+ auto CachedIt = FunctionBlockFreqInfoCache.find(F);
+ if (CachedIt == FunctionBlockFreqInfoCache.end())
+ return;
+ const AllBlockFreqInfo &AllFreqInfo = CachedIt->second;
+
+ for (const BasicBlock &BB : *F) {
+ const Instruction *Term = BB.getTerminator();
+ if (!Term)
+ continue;
+ if (Term->getNumSuccessors() == 0)
+ continue;
+
+ auto It = AllFreqInfo.find(&BB);
+ if (It == AllFreqInfo.end())
+ continue;
+
+ const BlockFreqInfo &Info = It->second;
+ // Only diagnose hard mismatches once both sides are fully known; otherwise
+ // leave the block as debug-only inconclusive state.
+ if (Info.numUnknownIn == 0 && Info.numUnknownOut == 0 &&
+ Info.sumIn != Info.sumOut) {
+ if (VerifyIPGOPrintDiagnostics)
+ errs() << "PGOVerify# Block frequency mismatch in function "
+ << F->getName() << ", block " << BB.getName()
+ << ": Incoming=" << Info.sumIn
+ << ": Outgoing=" << Info.sumOut << "\n";
+ LLVM_DEBUG(dbgs() << "PGOVerify# Block frequency mismatch in function "
+ << F->getName() << ", block " << BB.getName()
+ << ": Incoming=" << Info.sumIn
+ << ": Outgoing=" << Info.sumOut << "\n");
+ } else if (Info.numUnknownIn != 0 || Info.numUnknownOut != 0) {
+ LLVM_DEBUG(
+ dbgs() << "PGOVerify# Not able to determine Block frequency for "
+ << F->getName() << ", block " << BB.getName() << "\n");
+ }
+ }
+}
diff --git a/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-edge-cases.proftext b/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-edge-cases.proftext
new file mode 100644
index 0000000000000..33863b766852c
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-edge-cases.proftext
@@ -0,0 +1,60 @@
+# IR level Instrumentation Flag
+:ir
+multiple_returns
+# Func Hash:
+238984481941143025
+# Num Counters:
+3
+# Counter Values:
+1000
+200
+500
+
+nested_correct
+# Func Hash:
+536873292852368260
+# Num Counters:
+3
+# Counter Values:
+1000
+700
+400
+
+nested_incorrect
+# Func Hash:
+536873292852368260
+# Num Counters:
+3
+# Counter Values:
+1000
+700
+300
+
+switch_all_returns
+# Func Hash:
+1124680651005410801
+# Num Counters:
+4
+# Counter Values:
+1000
+200
+300
+400
+
+with_assertion
+# Func Hash:
+784007058953177093
+# Num Counters:
+2
+# Counter Values:
+1000
+999
+
+with_unreachable
+# Func Hash:
+1063705161907589342
+# Num Counters:
+2
+# Counter Values:
+1000
+600
diff --git a/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-flow-conservation.proftext b/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-flow-conservation.proftext
new file mode 100644
index 0000000000000..d9e3257815c78
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-block-flow-conservation.proftext
@@ -0,0 +1,55 @@
+# IR level Instrumentation Flag
+:ir
+correct_if_else
+# Func Hash:
+146835647075900052
+# Num Counters:
+3
+# Counter Values:
+1000
+600
+400
+
+correct_loop
+# Func Hash:
+784007056507080604
+# Num Counters:
+3
+# Counter Values:
+10
+90
+10
+
+incorrect_diamond
+# Func Hash:
+1124680652064592474
+# Num Counters:
+5
+# Counter Values:
+1000
+400
+600
+700
+200
+
+incorrect_if_else_middle
+# Func Hash:
+146835647075900052
+# Num Counters:
+4
+# Counter Values:
+1000
+700
+500
+300
+
+incorrect_loop_flow
+# Func Hash:
+1063705162469825436
+# Num Counters:
+4
+# Counter Values:
+10
+70
+10
+80
diff --git a/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-mother-block-frequency.proftext b/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-mother-block-frequency.proftext
new file mode 100644
index 0000000000000..48659b3c6b1ce
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/Inputs/verify-ipgo-mother-block-frequency.proftext
@@ -0,0 +1,57 @@
+# IR level Instrumentation Flag
+:ir
+consistent_flow
+# Func Hash:
+146835647075900052
+# Num Counters:
+3
+# Counter Values:
+1000
+600
+400
+
+inconsistent_entry
+# Func Hash:
+784007058953177093
+# Num Counters:
+3
+# Counter Values:
+1000
+700
+200
+
+inconsistent_branches
+# Func Hash:
+536873293986372656
+# Num Counters:
+6
+# Counter Values:
+1000
+600
+400
+700
+200
+300
+
+inconsistent_loop
+# Func Hash:
+146835646621254984
+# Num Counters:
+4
+# Counter Values:
+100
+900
+800
+100
+
+inconsistent_switch
+# Func Hash:
+1124680651005410801
+# Num Counters:
+5
+# Counter Values:
+1000
+200
+300
+300
+100
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-edge-cases.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-edge-cases.ll
new file mode 100644
index 0000000000000..48c8aeb47e746
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-edge-cases.ll
@@ -0,0 +1,205 @@
+; REQUIRES: asserts
+; RUN: llvm-profdata merge %S/Inputs/verify-ipgo-block-edge-cases.proftext -o %t.profdata
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck --check-prefix=VERIFY %s
+
+; Mother-patch edge cases for block-frequency validation.
+; Ensures verifier skips exit/unreachable-only blocks and reports real mismatches.
+
+define i32 @multiple_returns(i32 %x) !prof !5 {
+entry:
+ %cmp1 = icmp eq i32 %x, 0
+ br i1 %cmp1, label %return.zero, label %check.positive, !prof !6
+
+check.positive:
+ %cmp2 = icmp sgt i32 %x, 0
+ br i1 %cmp2, label %return.positive, label %return.negative, !prof !7
+
+return.zero:
+ ret i32 0
+
+return.positive:
+ ret i32 1
+
+return.negative:
+ ret i32 -1
+}
+
+define i32 @with_unreachable(i32 %x) !prof !8 {
+entry:
+ %cmp = icmp sgt i32 %x, 100
+ br i1 %cmp, label %normal.path, label %also.normal, !prof !9
+
+normal.path:
+ ret i32 1
+
+also.normal:
+ ret i32 2
+
+dead.block:
+ %mul = mul i32 %x, 2
+ br label %more.dead
+
+more.dead:
+ unreachable
+}
+
+define i32 @nested_correct(i32 %a, i32 %b, i32 %c) !prof !10 {
+entry:
+ %cmp1 = icmp sgt i32 %a, 0
+ br i1 %cmp1, label %outer.then, label %outer.else, !prof !11
+
+outer.then:
+ %cmp2 = icmp sgt i32 %b, 0
+ br i1 %cmp2, label %inner.then, label %inner.else, !prof !12
+
+inner.then:
+ %add1 = add i32 %a, %b
+ br label %join.inner
+
+inner.else:
+ %sub1 = sub i32 %a, %b
+ br label %join.inner
+
+join.inner:
+ %result1 = phi i32 [ %add1, %inner.then ], [ %sub1, %inner.else ]
+ br label %join.outer
+
+outer.else:
+ %mul = mul i32 %a, %c
+ br label %join.outer
+
+join.outer:
+ %final = phi i32 [ %result1, %join.inner ], [ %mul, %outer.else ]
+ ret i32 %final
+}
+
+define i32 @nested_incorrect(i32 %a, i32 %b, i32 %c) !prof !13 {
+entry:
+ %cmp1 = icmp sgt i32 %a, 0
+ br i1 %cmp1, label %outer.then, label %outer.else, !prof !14
+
+outer.then:
+ %cmp2 = icmp sgt i32 %b, 0
+ br i1 %cmp2, label %inner.then, label %inner.else, !prof !15
+
+inner.then:
+ %add1 = add i32 %a, %b
+ br label %join.inner
+
+inner.else:
+ %sub1 = sub i32 %a, %b
+ br label %join.inner
+
+join.inner:
+ %result1 = phi i32 [ %add1, %inner.then ], [ %sub1, %inner.else ]
+ br label %join.outer
+
+outer.else:
+ %mul = mul i32 %a, %c
+ br label %join.outer
+
+join.outer:
+ %final = phi i32 [ %result1, %join.inner ], [ %mul, %outer.else ]
+ ret i32 %final
+}
+
+define i32 @switch_all_returns(i32 %x) !prof !20 {
+entry:
+ switch i32 %x, label %default [
+ i32 1, label %case1
+ i32 2, label %case2
+ i32 3, label %case3
+ ], !prof !21
+
+case1:
+ ret i32 10
+
+case2:
+ ret i32 20
+
+case3:
+ ret i32 30
+
+default:
+ ret i32 0
+}
+
+define i32 @with_assertion(i32 %x) !prof !22 {
+entry:
+ %valid = icmp sge i32 %x, 0
+ br i1 %valid, label %normal, label %error, !prof !23
+
+normal:
+ %result = mul i32 %x, 2
+ ret i32 %result
+
+error:
+ call void @abort() noreturn
+ unreachable
+}
+
+declare void @abort() noreturn
+
+; Function with manually set inconsistent branch weights, intentionally not listed in
+; the proftext so pgo-instr-use will leave its metadata unchanged.
+; outer.then: incoming=700 (from entry branch weight), outgoing=300+300=600 → mismatch.
+define i32 @nested_inconsistent_manual(i32 %a, i32 %b) !prof !24 {
+entry:
+ %cmp1 = icmp sgt i32 %a, 0
+ br i1 %cmp1, label %outer.then, label %outer.else, !prof !25
+
+outer.then:
+ %cmp2 = icmp sgt i32 %b, 0
+ br i1 %cmp2, label %inner.then, label %inner.else, !prof !26
+
+inner.then:
+ ret i32 1
+
+inner.else:
+ ret i32 0
+
+outer.else:
+ ret i32 -1
+}
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+!llvm.ident = !{!4}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+
+!5 = !{!"function_entry_count", i64 1000}
+!6 = !{!"branch_weights", i32 200, i32 800}
+!7 = !{!"branch_weights", i32 500, i32 300}
+
+!8 = !{!"function_entry_count", i64 1000}
+!9 = !{!"branch_weights", i32 600, i32 400}
+
+!10 = !{!"function_entry_count", i64 1000}
+!11 = !{!"branch_weights", i32 700, i32 300}
+!12 = !{!"branch_weights", i32 400, i32 300}
+
+!13 = !{!"function_entry_count", i64 1000}
+!14 = !{!"branch_weights", i32 700, i32 300}
+!15 = !{!"branch_weights", i32 300, i32 400}
+
+!20 = !{!"function_entry_count", i64 1000}
+!21 = !{!"branch_weights", i32 100, i32 200, i32 300, i32 400}
+
+!22 = !{!"function_entry_count", i64 1000}
+!23 = !{!"branch_weights", i32 999, i32 1}
+
+!24 = !{!"function_entry_count", i64 1000}
+!25 = !{!"branch_weights", i32 700, i32 300}
+!26 = !{!"branch_weights", i32 300, i32 300}
+
+; CHECK: *** IPGO Verification After PGOInstrumentationUse ***
+; CHECK: PGOVerify cache invalidated
+; CHECK: PGOVerify# Block frequency mismatch in function nested_inconsistent_manual, block outer.then: Incoming=700: Outgoing=600
+
+; VERIFY: *** IPGO Verification After PGOInstrumentationUse ***
+; VERIFY: PGOVerify# Block frequency mismatch in function nested_inconsistent_manual, block outer.then: Incoming=700: Outgoing=600
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-flow-conservation.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-flow-conservation.ll
new file mode 100644
index 0000000000000..dd2de24a82093
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-flow-conservation.ll
@@ -0,0 +1,56 @@
+; REQUIRES: asserts
+; RUN: opt -debug-only=verify-ipgo -verify-ipgo -passes='instcombine' -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -verify-ipgo -passes='instcombine' -disable-output %s 2>&1 | FileCheck %s --check-prefix=VERIFY
+;
+; flow-conservation test, intentionally checking only
+; block-frequency mismatch diagnostics.
+
+define i32 @incorrect_if_else_middle(i32 %x) !prof !0 {
+entry:
+ %cmp = icmp sgt i32 %x, 0
+ br i1 %cmp, label %if.then, label %if.else, !prof !1
+
+if.then:
+ %add = add i32 %x, 5
+ %cmp2 = icmp sgt i32 %add, 0
+ br i1 %cmp2, label %merge, label %if.then.cont, !prof !2
+
+if.then.cont:
+ br label %merge
+
+if.else:
+ %sub = sub i32 %x, 5
+ br label %merge
+
+merge:
+ %val = phi i32 [ %add, %if.then ], [ %add, %if.then.cont ], [ %sub, %if.else ]
+ %mul = mul i32 %val, 2
+ ret i32 %mul
+}
+
+; CHECK: *** IPGO Verification After InstCombinePass ***
+; CHECK: PGOVerify cache invalidated
+; CHECK: PGOVerify# Block frequency mismatch in function incorrect_if_else_middle, block if.then: Incoming=700: Outgoing=600
+
+; VERIFY: *** IPGO Verification After InstCombinePass ***
+; VERIFY: PGOVerify# Block frequency mismatch in function incorrect_if_else_middle, block if.then: Incoming=700: Outgoing=600
+
+!0 = !{!"function_entry_count", i64 1000}
+!1 = !{!"branch_weights", i32 700, i32 300}
+!2 = !{!"branch_weights", i32 400, i32 200}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 1000}
+!14 = !{!"MaxCount", i64 700}
+!15 = !{!"MaxInternalCount", i64 700}
+!16 = !{!"MaxFunctionCount", i64 1000}
+!17 = !{!"NumCounts", i64 3}
+!18 = !{!"NumFunctions", i64 1}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21, !22, !23}
+!21 = !{i32 10000, i64 700, i32 1}
+!22 = !{i32 999000, i64 500, i32 2}
+!23 = !{i32 999999, i64 300, i32 3}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-frequency-mismatch.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-frequency-mismatch.ll
new file mode 100644
index 0000000000000..cdca0b3bd62d1
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-block-frequency-mismatch.ll
@@ -0,0 +1,135 @@
+; REQUIRES: asserts
+; RUN: opt -debug-only=verify-ipgo -verify-ipgo -passes='instcombine' -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -verify-ipgo -passes='instcombine' -disable-output %s 2>&1 | FileCheck %s --check-prefix=VERIFY
+;
+; Mother-patch-derived block-frequency test, intentionally checking only
+; block-frequency mismatch diagnostics.
+
+define i32 @inconsistent_entry(i32 %x) !prof !0 {
+entry:
+ %cmp = icmp sgt i32 %x, 0
+ br i1 %cmp, label %positive, label %negative, !prof !1
+
+positive:
+ %mul = mul nsw i32 %x, 2
+ %sink1 = add nsw i32 %mul, 0
+ ret i32 %sink1
+
+negative:
+ %div = sdiv i32 %x, 2
+ %sink2 = add nsw i32 %div, 0
+ ret i32 %sink2
+}
+
+define i32 @inconsistent_loop(i32 %n) !prof !2 {
+entry:
+ br label %loop.header
+
+loop.header:
+ %i = phi i32 [ 0, %entry ], [ %inc, %loop.body ]
+ %sum = phi i32 [ 0, %entry ], [ %add, %loop.body ]
+ %cmp = icmp slt i32 %i, %n
+ br i1 %cmp, label %loop.body, label %loop.exit, !prof !3
+
+loop.body:
+ %add = add nsw i32 %sum, %i
+ %inc = add nsw i32 %i, 1
+ br label %loop.header
+
+loop.exit:
+ ret i32 %sum
+}
+
+define i32 @inconsistent_branches(i32 %a, i32 %b) !prof !5 {
+entry:
+ %cmp1 = icmp sgt i32 %a, 0
+ br i1 %cmp1, label %then1, label %else1, !prof !6
+
+then1:
+ %mul = mul nsw i32 %a, 2
+ br label %middle
+
+else1:
+ %div = sdiv i32 %a, 2
+ br label %middle
+
+middle:
+ %val = phi i32 [ %mul, %then1 ], [ %div, %else1 ]
+ %cmp2 = icmp sgt i32 %b, 0
+ br i1 %cmp2, label %then2, label %else2, !prof !9
+
+then2:
+ %add = add nsw i32 %val, %b
+ br label %end
+
+else2:
+ %sub = sub nsw i32 %val, %b
+ br label %end
+
+end:
+ %result = phi i32 [ %add, %then2 ], [ %sub, %else2 ]
+ ret i32 %result
+}
+
+define i32 @inconsistent_switch(i32 %x) !prof !12 {
+entry:
+ switch i32 %x, label %default [
+ i32 1, label %case1
+ i32 2, label %case2
+ i32 3, label %case3
+ ], !prof !13
+
+case1:
+ ret i32 10
+
+case2:
+ ret i32 20
+
+case3:
+ ret i32 30
+
+default:
+ ret i32 0
+}
+
+; CHECK: *** IPGO Verification After InstCombinePass ***
+; CHECK: PGOVerify cache invalidated
+; CHECK: PGOVerify# Block frequency mismatch in function inconsistent_entry, block entry: Incoming=1000: Outgoing=900
+; CHECK: PGOVerify# Block frequency mismatch in function inconsistent_branches, block middle: Incoming=1000: Outgoing=900
+; CHECK: PGOVerify# Block frequency mismatch in function inconsistent_switch, block entry: Incoming=1000: Outgoing=900
+
+; VERIFY: *** IPGO Verification After InstCombinePass ***
+; VERIFY: PGOVerify# Block frequency mismatch in function inconsistent_entry, block entry: Incoming=1000: Outgoing=900
+; VERIFY: PGOVerify# Block frequency mismatch in function inconsistent_branches, block middle: Incoming=1000: Outgoing=900
+; VERIFY: PGOVerify# Block frequency mismatch in function inconsistent_switch, block entry: Incoming=1000: Outgoing=900
+
+!0 = !{!"function_entry_count", i64 1000}
+!1 = !{!"branch_weights", i32 700, i32 200}
+!2 = !{!"function_entry_count", i64 100}
+!3 = !{!"branch_weights", i32 900, i32 100}
+!4 = !{!"branch_weights", i32 800}
+!5 = !{!"function_entry_count", i64 1000}
+!6 = !{!"branch_weights", i32 600, i32 400}
+!7 = !{!"branch_weights", i32 600}
+!8 = !{!"branch_weights", i32 400}
+!9 = !{!"branch_weights", i32 700, i32 200}
+!10 = !{!"branch_weights", i32 700}
+!11 = !{!"branch_weights", i32 200}
+!12 = !{!"function_entry_count", i64 1000}
+!13 = !{!"branch_weights", i32 100, i32 200, i32 300, i32 300}
+
+!llvm.module.flags = !{!20}
+!20 = !{i32 1, !"ProfileSummary", !21}
+!21 = !{!22, !23, !24, !25, !26, !27, !28, !29}
+!22 = !{!"ProfileFormat", !"InstrProf"}
+!23 = !{!"TotalCount", i64 4000}
+!24 = !{!"MaxCount", i64 1000}
+!25 = !{!"MaxInternalCount", i64 1000}
+!26 = !{!"MaxFunctionCount", i64 1000}
+!27 = !{!"NumCounts", i64 13}
+!28 = !{!"NumFunctions", i64 4}
+!29 = !{!"DetailedSummary", !30}
+!30 = !{!31, !32, !33}
+!31 = !{i32 10000, i64 1000, i32 1}
+!32 = !{i32 999000, i64 900, i32 4}
+!33 = !{i32 999999, i64 100, i32 13}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-cache-invalidation.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-cache-invalidation.ll
new file mode 100644
index 0000000000000..71414c57c47a2
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-cache-invalidation.ll
@@ -0,0 +1,17 @@
+; REQUIRES: asserts
+; RUN: opt -debug-only=verify-ipgo -passes='instcombine,instcombine' -verify-ipgo -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -passes='instcombine,instcombine' -verify-ipgo -disable-output %s 2>&1 | FileCheck %s --check-prefix=VERIFY
+;
+; Verify cache invalidation is emitted for function- and module-level IR units.
+;
+; CHECK: PGOVerify cache invalidated: function
+; CHECK: PGOVerify cache invalidated: module
+
+; VERIFY: *** IPGO Verification After InstCombinePass ***
+; VERIFY: *** IPGO Verification After InstCombinePass (Skipped) ***
+
+define i32 @f(i32 %x) {
+entry:
+ %a = add i32 %x, 0
+ ret i32 %a
+}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-cfg-transform-no-false-positives.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-cfg-transform-no-false-positives.ll
new file mode 100644
index 0000000000000..a509d750c4e7a
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-cfg-transform-no-false-positives.ll
@@ -0,0 +1,88 @@
+; REQUIRES: asserts
+; RUN: llvm-profdata merge %S/Inputs/verify-ipgo-block-flow-conservation.proftext -o %t.profdata
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-use,simplifycfg -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s --check-prefix=SIMPLIFY --implicit-check-not="PGOVerify# Block frequency mismatch"
+; RUN: opt < %s -verify-ipgo -passes=pgo-instr-use,simplifycfg -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s --check-prefix=SIMPLIFY-VERIFY --implicit-check-not="PGOVerify# Block frequency mismatch"
+; RUN: llvm-profdata merge %S/Inputs/verify-ipgo-block-flow-conservation.proftext -o %t.profdata
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-use,jump-threading -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s --check-prefix=JTHREAD --implicit-check-not="PGOVerify# Block frequency mismatch"
+; RUN: opt < %s -verify-ipgo -passes=pgo-instr-use,jump-threading -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s --check-prefix=JTHREAD-VERIFY --implicit-check-not="PGOVerify# Block frequency mismatch"
+
+; Mother-patch regression for CFG-changing transforms:
+; verify-ipgo should not report block-flow mismatches after simplifycfg/jump-threading
+; when flow is profile-consistent.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @cfg_simplify(i32 %x) !prof !0 {
+entry:
+ %cmp = icmp sgt i32 %x, 0
+ br i1 %cmp, label %then, label %else, !prof !1
+
+then:
+ br label %join
+
+else:
+ br label %join
+
+join:
+ %v = phi i32 [ 1, %then ], [ 1, %else ]
+ br i1 true, label %ret, label %dead, !prof !4
+
+dead:
+ unreachable
+
+ret:
+ ret i32 %v
+}
+
+define i32 @cfg_thread(i1 %c) !prof !5 {
+entry:
+ br i1 %c, label %left, label %right, !prof !6
+
+left:
+ br label %merge
+
+right:
+ br label %merge
+
+merge:
+ %p = phi i1 [ true, %left ], [ false, %right ]
+ br i1 %p, label %taken, label %nottaken, !prof !9
+
+taken:
+ ret i32 1
+
+nottaken:
+ ret i32 0
+}
+
+!llvm.module.flags = !{!10, !11}
+!llvm.ident = !{!12}
+
+!0 = !{!"function_entry_count", i64 1000}
+!1 = !{!"branch_weights", i32 600, i32 400}
+!2 = !{!"branch_weights", i32 600}
+!3 = !{!"branch_weights", i32 400}
+!4 = !{!"branch_weights", i32 1000, i32 0}
+
+!5 = !{!"function_entry_count", i64 1000}
+!6 = !{!"branch_weights", i32 700, i32 300}
+!7 = !{!"branch_weights", i32 700}
+!8 = !{!"branch_weights", i32 300}
+!9 = !{!"branch_weights", i32 700, i32 300}
+
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{i32 7, !"uwtable", i32 2}
+!12 = !{!"clang version 21.1.8"}
+
+; SIMPLIFY: *** IPGO Verification After PGOInstrumentationUse ***
+; SIMPLIFY: *** IPGO Verification After SimplifyCFGPass ***
+
+; SIMPLIFY-VERIFY: *** IPGO Verification After PGOInstrumentationUse ***
+; SIMPLIFY-VERIFY: *** IPGO Verification After SimplifyCFGPass ***
+
+; JTHREAD: *** IPGO Verification After PGOInstrumentationUse ***
+; JTHREAD: *** IPGO Verification After JumpThreadingPass ***
+
+; JTHREAD-VERIFY: *** IPGO Verification After PGOInstrumentationUse ***
+; JTHREAD-VERIFY: *** IPGO Verification After JumpThreadingPass ***
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-cs-instr-summary-overflow-skip.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-cs-instr-summary-overflow-skip.ll
new file mode 100644
index 0000000000000..eec7b88aaf671
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-cs-instr-summary-overflow-skip.ll
@@ -0,0 +1,52 @@
+; REQUIRES: asserts
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+;
+; Coverage test for hasInstrProfUseSummary:
+; - Uses CS profile summary (queried first via getProfileSummary(true)).
+; - Current behavior does not emit unknown block-frequency diagnostics for this IR.
+
+define internal i32 @cs_summary_loop(i32 %n) !prof !10 {
+entry:
+ %x = add i32 %n, 0
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %inc, %latch ]
+ %cmp = icmp slt i32 %i, %x
+ br i1 %cmp, label %body, label %exit
+
+body:
+ br label %latch
+
+latch:
+ %inc = add i32 %i, 1
+ br label %header
+
+exit:
+ ret i32 %i
+}
+
+; CHECK-LABEL: *** IPGO Verification After InstCombinePass ***
+; CHECK-NOT: PGOVerify# Not able to determine Block frequency for cs_summary_loop, block header
+
+; VERIFY-LABEL: *** IPGO Verification After InstCombinePass ***
+; VERIFY-NOT: PGOVerify# Not able to determine Block frequency for cs_summary_loop, block header
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"CSProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"CSInstrProf"}
+!33 = !{!"TotalCount", i64 4294967297}
+!34 = !{!"MaxCount", i64 4294967296}
+!35 = !{!"MaxInternalCount", i64 4294967296}
+!36 = !{!"MaxFunctionCount", i64 4294967296}
+!37 = !{!"NumCounts", i64 1}
+!38 = !{!"NumFunctions", i64 1}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 4294967296, i32 1}
+!42 = !{i32 999000, i64 4294967296, i32 1}
+!43 = !{i32 999999, i64 1, i32 1}
+
+!10 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll
new file mode 100644
index 0000000000000..d790da8c17fde
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-gen -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=pgo-instr-gen -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+;
+; Ensure verify-ipgo runs in Gen phase without emitting entry/block diagnostics
+; for this minimal IR.
+
+ at __profc_bar = global i64 0, align 8
+
+define i32 @foo(i32 %x) {
+entry:
+ %v = load i64, ptr @__profc_bar, align 8
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+; CHECK-LABEL: *** IPGO Verification After PGOInstrumentationGen ***
+; CHECK-NOT: PGOVerify# Entry count mismatch
+; CHECK-NOT: PGOVerify# Block frequency mismatch
+
+; VERIFY-LABEL: *** IPGO Verification After PGOInstrumentationGen ***
+; VERIFY-NOT: PGOVerify# Entry count mismatch
+; VERIFY-NOT: PGOVerify# Block frequency mismatch
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-large-entry-overflow-skip.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-large-entry-overflow-skip.ll
new file mode 100644
index 0000000000000..aa02df6e14f96
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-large-entry-overflow-skip.ll
@@ -0,0 +1,82 @@
+; REQUIRES: asserts
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+;
+; Mother-patch overflow gating test (trimmed to block-frequency diagnostics).
+; - overflow_loop has entry count > uint32 max and should be skipped.
+; - normal_loop has small entry count and should still emit unknown-frequency
+; diagnostics in this synthetic setup.
+
+define internal i32 @overflow_loop(i32 %n) !prof !10 {
+entry:
+ %x = add i32 %n, 0
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %inc, %latch ]
+ %cmp = icmp slt i32 %i, %x
+ br i1 %cmp, label %body, label %exit
+
+body:
+ br label %latch
+
+latch:
+ %inc = add i32 %i, 1
+ br label %header
+
+exit:
+ ret i32 %i
+}
+
+define internal i32 @normal_loop(i32 %n) !prof !11 {
+entry:
+ %x = add i32 %n, 0
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %inc, %latch ]
+ %cmp = icmp slt i32 %i, %x
+ br i1 %cmp, label %body, label %exit
+
+body:
+ br label %latch
+
+latch:
+ %inc = add i32 %i, 1
+ br label %header
+
+exit:
+ ret i32 %i
+}
+
+define i32 @main() {
+entry:
+ %a = call i32 @overflow_loop(i32 4)
+ %b = call i32 @normal_loop(i32 4)
+ %s = add i32 %a, %b
+ ret i32 %s
+}
+
+; CHECK-NOT: PGOVerify# Not able to determine Block frequency for overflow_loop
+; CHECK: PGOVerify# Not able to determine Block frequency for normal_loop, block header
+
+; VERIFY: *** IPGO Verification After InstCombinePass ***
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 4294967297}
+!34 = !{!"MaxCount", i64 4294967296}
+!35 = !{!"MaxInternalCount", i64 4294967296}
+!36 = !{!"MaxFunctionCount", i64 4294967296}
+!37 = !{!"NumCounts", i64 2}
+!38 = !{!"NumFunctions", i64 2}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 4294967296, i32 1}
+!42 = !{i32 999000, i64 4294967296, i32 1}
+!43 = !{i32 999999, i64 1, i32 2}
+
+!10 = !{!"function_entry_count", i64 4294967296}
+!11 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-loop-rotate.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-loop-rotate.ll
new file mode 100644
index 0000000000000..3467c3cad028e
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-loop-rotate.ll
@@ -0,0 +1,306 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=loop-rotate -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=loop-rotate -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+;
+; Reduced reproducer from build/looprotate.ll.
+; This verifies that loop-rotate currently triggers block-flow mismatch
+; diagnostics in verify-ipgo for update_tree.
+
+source_filename = "treeup.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+%struct.arc = type { i32, i64, ptr, ptr, i16, ptr, ptr, i64, i64 }
+%struct.node = type { i64, i32, ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr, i64, i64, i32, i32 }
+; Function Attrs: nounwind uwtable
+define dso_local void @update_tree(i64 noundef %cycle_ori, i64 noundef %new_orientation, i64 noundef %delta, i64 noundef %new_flow, ptr noundef %iplus, ptr noundef %jplus, ptr noundef captures(address) %iminus, ptr noundef captures(address) %jminus, ptr noundef readnone captures(
+address) %w, ptr noundef %bea, i64 noundef %sigma, i64 noundef %feas_tol) local_unnamed_addr #0 !prof !34 {
+entry:
+ %tail = getelementptr inbounds nuw %struct.arc, ptr %bea, i64 0, i32 2
+ %0 = load ptr, ptr %tail, align 8
+ %cmp = icmp eq ptr %0, %jplus
+ %cmp1 = icmp slt i64 %sigma, 0
+ %or.cond = and i1 %cmp1, %cmp
+ br i1 %or.cond, label %if.then, label %lor.lhs.false, !prof !45
+lor.lhs.false: ; preds = %entry
+ %cmp3 = icmp eq ptr %0, %iplus
+ %cmp5 = icmp sgt i64 %sigma, 0
+ %or.cond224 = and i1 %cmp5, %cmp3
+ br i1 %or.cond224, label %if.then, label %if.else, !prof !46
+if.then: ; preds = %lor.lhs.false, %entry
+ %cond = tail call i64 @llvm.abs.i64(i64 %sigma, i1 true)
+ br label %if.end
+if.else: ; preds = %lor.lhs.false
+ %cond12 = tail call i64 @llvm.abs.i64(i64 %sigma, i1 true)
+ %sub13 = sub nsw i64 0, %cond12
+ br label %if.end
+if.end: ; preds = %if.else, %if.then
+ %sigma.addr.0 = phi i64 [ %cond, %if.then ], [ %sub13, %if.else ]
+ %1 = load i64, ptr %iminus, align 8
+ %add = add nsw i64 %1, %sigma.addr.0
+ store i64 %add, ptr %iminus, align 8
+ br label %RECURSION
+RECURSION: ; preds = %ITERATION, %if.end
+ %father.0 = phi ptr [ %iminus, %if.end ], [ %temp.0, %ITERATION ]
+ %child = getelementptr inbounds nuw %struct.node, ptr %father.0, i64 0, i32 2
+ %2 = load ptr, ptr %child, align 8
+ %tobool.not = icmp eq ptr %2, null
+ br i1 %tobool.not, label %TEST.preheader, label %ITERATION, !prof !50
+TEST.preheader: ; preds = %RECURSION
+ br label %TEST
+ITERATION.loopexit: ; preds = %if.end20
+ %.lcssa = phi ptr [ %4, %if.end20 ]
+ br label %ITERATION
+ITERATION: ; preds = %ITERATION.loopexit, %RECURSION
+ %temp.0 = phi ptr [ %2, %RECURSION ], [ %.lcssa, %ITERATION.loopexit ]
+ %3 = load i64, ptr %temp.0, align 8
+ %add16 = add nsw i64 %3, %sigma.addr.0
+ store i64 %add16, ptr %temp.0, align 8
+ br label %RECURSION
+TEST: ; preds = %TEST.preheader, %if.end23
+ %father.1 = phi ptr [ %5, %if.end23 ], [ %father.0, %TEST.preheader ]
+ %cmp18 = icmp eq ptr %father.1, %iminus
+ br i1 %cmp18, label %CONTINUE, label %if.end20, !prof !51
+if.end20: ; preds = %TEST
+ %sibling = getelementptr inbounds nuw %struct.node, ptr %father.1, i64 0, i32 4
+ %4 = load ptr, ptr %sibling, align 8
+ %tobool21.not = icmp eq ptr %4, null
+ br i1 %tobool21.not, label %if.end23, label %ITERATION.loopexit, !prof !53
+if.end23: ; preds = %if.end20
+ %pred = getelementptr inbounds nuw %struct.node, ptr %father.1, i64 0, i32 3
+ %5 = load ptr, ptr %pred, align 8
+ br label %TEST
+CONTINUE: ; preds = %TEST
+ %pred24 = getelementptr inbounds nuw %struct.node, ptr %iplus, i64 0, i32 3
+ %6 = load ptr, ptr %pred24, align 8
+ %depth = getelementptr inbounds nuw %struct.node, ptr %iminus, i64 0, i32 11
+ %7 = load i64, ptr %depth, align 8
+ br label %while.cond
+while.cond: ; preds = %if.end61, %CONTINUE
+ %new_basic_arc.0 = phi ptr [ %bea, %CONTINUE ], [ %15, %if.end61 ]
+ %father.2 = phi ptr [ %6, %CONTINUE ], [ %17, %if.end61 ]
+ %temp.1 = phi ptr [ %iplus, %CONTINUE ], [ %father.2, %if.end61 ]
+ %new_pred.0 = phi ptr [ %jplus, %CONTINUE ], [ %temp.1, %if.end61 ]
+ %new_flow.addr.0 = phi i64 [ %new_flow, %CONTINUE ], [ %flow_temp.0, %if.end61 ]
+ %new_orientation.addr.0 = phi i64 [ %new_orientation, %CONTINUE ], [ %conv, %if.end61 ]
+ %new_depth.0 = phi i64 [ %7, %CONTINUE ], [ %sub68, %if.end61 ]
+ %cmp25.not = icmp eq ptr %temp.1, %jminus
+ br i1 %cmp25.not, label %while.end, label %while.body, !prof !56
+while.body: ; preds = %while.cond
+ %sibling26 = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 4
+ %8 = load ptr, ptr %sibling26, align 8
+ %tobool27.not = icmp eq ptr %8, null
+ br i1 %tobool27.not, label %if.end31, label %if.then28, !prof !57
+if.then28: ; preds = %while.body
+ %sibling_prev = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 5
+ %9 = load ptr, ptr %sibling_prev, align 8
+ %sibling_prev30 = getelementptr inbounds nuw %struct.node, ptr %8, i64 0, i32 5
+ store ptr %9, ptr %sibling_prev30, align 8
+ br label %if.end31
+if.end31: ; preds = %if.then28, %while.body
+ %sibling_prev32 = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 5
+ %10 = load ptr, ptr %sibling_prev32, align 8
+ %tobool33.not = icmp eq ptr %10, null
+ br i1 %tobool33.not, label %if.else38, label %if.then34, !prof !59
+if.then34: ; preds = %if.end31
+ %sibling37 = getelementptr inbounds nuw %struct.node, ptr %10, i64 0, i32 4
+ store ptr %8, ptr %sibling37, align 8
+ br label %if.end41
+if.else38: ; preds = %if.end31
+ %child40 = getelementptr inbounds nuw %struct.node, ptr %father.2, i64 0, i32 2
+ store ptr %8, ptr %child40, align 8
+ br label %if.end41
+if.end41: ; preds = %if.else38, %if.then34
+ %pred42 = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 3
+ store ptr %new_pred.0, ptr %pred42, align 8
+ %child43 = getelementptr inbounds nuw %struct.node, ptr %new_pred.0, i64 0, i32 2
+ %11 = load ptr, ptr %child43, align 8
+ store ptr %11, ptr %sibling26, align 8
+ %tobool46.not = icmp eq ptr %11, null
+ br i1 %tobool46.not, label %if.end50, label %if.then47, !prof !60
+if.then47: ; preds = %if.end41
+ %sibling_prev49 = getelementptr inbounds nuw %struct.node, ptr %11, i64 0, i32 5
+ store ptr %temp.1, ptr %sibling_prev49, align 8
+ br label %if.end50
+if.end50: ; preds = %if.then47, %if.end41
+ store ptr %temp.1, ptr %child43, align 8
+ store ptr null, ptr %sibling_prev32, align 8
+ %orientation = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 1
+ %12 = load i32, ptr %orientation, align 8
+ %tobool53.not = icmp eq i32 %12, 0
+ %conv = zext i1 %tobool53.not to i64
+ %cmp54 = icmp eq i64 %cycle_ori, %conv
+ br i1 %cmp54, label %if.then56, label %if.else58, !prof !62
+if.then56: ; preds = %if.end50
+ %flow = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 10
+ %13 = load i64, ptr %flow, align 8
+ %add57 = add nsw i64 %13, %delta
+ br label %if.end61
+if.else58: ; preds = %if.end50
+ %flow59 = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 10
+ %14 = load i64, ptr %flow59, align 8
+ %sub60 = sub nsw i64 %14, %delta
+ br label %if.end61
+if.end61: ; preds = %if.else58, %if.then56
+ %flow_temp.0 = phi i64 [ %add57, %if.then56 ], [ %sub60, %if.else58 ]
+ %basic_arc = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 6
+ %15 = load ptr, ptr %basic_arc, align 8
+ %depth62 = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 11
+ %16 = load i64, ptr %depth62, align 8
+ %conv63 = trunc i64 %new_orientation.addr.0 to i32
+ store i32 %conv63, ptr %orientation, align 8
+ %flow65 = getelementptr inbounds nuw %struct.node, ptr %temp.1, i64 0, i32 10
+ store i64 %new_flow.addr.0, ptr %flow65, align 8
+ store ptr %new_basic_arc.0, ptr %basic_arc, align 8
+ store i64 %new_depth.0, ptr %depth62, align 8
+ %sub68 = sub nsw i64 %7, %16
+ %pred69 = getelementptr inbounds nuw %struct.node, ptr %father.2, i64 0, i32 3
+ %17 = load ptr, ptr %pred69, align 8
+ br label %while.cond, !llvm.loop !65
+while.end: ; preds = %while.cond
+ %cmp70 = icmp sgt i64 %delta, %feas_tol
+ br i1 %cmp70, label %for.cond.preheader, label %for.cond110.preheader, !prof !67
+for.cond110.preheader: ; preds = %while.end
+ br label %for.cond110
+for.cond.preheader: ; preds = %while.end
+ br label %for.cond
+for.cond: ; preds = %for.cond.preheader, %for.inc
+ %temp.2 = phi ptr [ %22, %for.inc ], [ %jminus, %for.cond.preheader ]
+ %cmp73.not = icmp eq ptr %temp.2, %w
+ br i1 %cmp73.not, label %for.cond89.preheader, label %for.body, !prof !68
+for.cond89.preheader: ; preds = %for.cond
+ br label %for.cond89
+for.body: ; preds = %for.cond
+ %depth75 = getelementptr inbounds nuw %struct.node, ptr %temp.2, i64 0, i32 11
+ %18 = load i64, ptr %depth75, align 8
+ %sub76 = sub nsw i64 %18, %7
+ store i64 %sub76, ptr %depth75, align 8
+ %orientation77 = getelementptr inbounds nuw %struct.node, ptr %temp.2, i64 0, i32 1
+ %19 = load i32, ptr %orientation77, align 8
+ %conv78 = sext i32 %19 to i64
+ %cmp79.not = icmp eq i64 %cycle_ori, %conv78
+ br i1 %cmp79.not, label %if.else84, label %if.then81
+if.then81: ; preds = %for.body
+ %flow82 = getelementptr inbounds nuw %struct.node, ptr %temp.2, i64 0, i32 10
+ %20 = load i64, ptr %flow82, align 8
+ %add83 = add nsw i64 %20, %delta
+ store i64 %add83, ptr %flow82, align 8
+ br label %for.inc
+if.else84: ; preds = %for.body
+ %flow85 = getelementptr inbounds nuw %struct.node, ptr %temp.2, i64 0, i32 10
+ %21 = load i64, ptr %flow85, align 8
+ %sub86 = sub nsw i64 %21, %delta
+ store i64 %sub86, ptr %flow85, align 8
+ br label %for.inc
+for.inc: ; preds = %if.then81, %if.else84
+ %pred88 = getelementptr inbounds nuw %struct.node, ptr %temp.2, i64 0, i32 3
+ %22 = load ptr, ptr %pred88, align 8
+ br label %for.cond, !llvm.loop !69
+for.cond89: ; preds = %for.cond89.preheader, %for.inc106
+ %temp.3 = phi ptr [ %27, %for.inc106 ], [ %jplus, %for.cond89.preheader ]
+ %cmp90.not = icmp eq ptr %temp.3, %w
+ br i1 %cmp90.not, label %if.end128.loopexit, label %for.body92, !prof !68
+for.body92: ; preds = %for.cond89
+ %depth93 = getelementptr inbounds nuw %struct.node, ptr %temp.3, i64 0, i32 11
+ %23 = load i64, ptr %depth93, align 8
+ %add94 = add nsw i64 %23, %7
+ store i64 %add94, ptr %depth93, align 8
+ %orientation95 = getelementptr inbounds nuw %struct.node, ptr %temp.3, i64 0, i32 1
+ %24 = load i32, ptr %orientation95, align 8
+ %conv96 = sext i32 %24 to i64
+ %cmp97 = icmp eq i64 %cycle_ori, %conv96
+ br i1 %cmp97, label %if.then99, label %if.else102
+if.then99: ; preds = %for.body92
+ %flow100 = getelementptr inbounds nuw %struct.node, ptr %temp.3, i64 0, i32 10
+ %25 = load i64, ptr %flow100, align 8
+ %add101 = add nsw i64 %25, %delta
+ store i64 %add101, ptr %flow100, align 8
+ br label %for.inc106
+if.else102: ; preds = %for.body92
+ %flow103 = getelementptr inbounds nuw %struct.node, ptr %temp.3, i64 0, i32 10
+ %26 = load i64, ptr %flow103, align 8
+ %sub104 = sub nsw i64 %26, %delta
+ store i64 %sub104, ptr %flow103, align 8
+ br label %for.inc106
+for.inc106: ; preds = %if.then99, %if.else102
+ %pred107 = getelementptr inbounds nuw %struct.node, ptr %temp.3, i64 0, i32 3
+ %27 = load ptr, ptr %pred107, align 8
+ br label %for.cond89, !llvm.loop !70
+for.cond110: ; preds = %for.cond110.preheader, %for.body113
+ %temp.4 = phi ptr [ %29, %for.body113 ], [ %jminus, %for.cond110.preheader ]
+ %cmp111.not = icmp eq ptr %temp.4, %w
+ br i1 %cmp111.not, label %for.cond119.preheader, label %for.body113, !prof !71
+for.cond119.preheader: ; preds = %for.cond110
+ br label %for.cond119
+for.body113: ; preds = %for.cond110
+ %depth114 = getelementptr inbounds nuw %struct.node, ptr %temp.4, i64 0, i32 11
+ %28 = load i64, ptr %depth114, align 8
+ %sub115 = sub nsw i64 %28, %7
+ store i64 %sub115, ptr %depth114, align 8
+ %pred117 = getelementptr inbounds nuw %struct.node, ptr %temp.4, i64 0, i32 3
+ %29 = load ptr, ptr %pred117, align 8
+ br label %for.cond110, !llvm.loop !72
+for.cond119: ; preds = %for.cond119.preheader, %for.body122
+ %temp.5 = phi ptr [ %31, %for.body122 ], [ %jplus, %for.cond119.preheader ]
+ %cmp120.not = icmp eq ptr %temp.5, %w
+ br i1 %cmp120.not, label %if.end128.loopexit225, label %for.body122, !prof !73
+for.body122: ; preds = %for.cond119
+ %depth123 = getelementptr inbounds nuw %struct.node, ptr %temp.5, i64 0, i32 11
+ %30 = load i64, ptr %depth123, align 8
+ %add124 = add nsw i64 %30, %7
+ store i64 %add124, ptr %depth123, align 8
+ %pred126 = getelementptr inbounds nuw %struct.node, ptr %temp.5, i64 0, i32 3
+ %31 = load ptr, ptr %pred126, align 8
+ br label %for.cond119, !llvm.loop !74
+if.end128.loopexit: ; preds = %for.cond89
+ br label %if.end128
+if.end128.loopexit225: ; preds = %for.cond119
+ br label %if.end128
+if.end128: ; preds = %if.end128.loopexit225, %if.end128.loopexit
+ ret void
+}
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i64 @llvm.abs.i64(i64, i1 immarg) #1
+
+!llvm.module.flags = !{!80}
+!80 = !{i32 1, !"ProfileSummary", !81}
+!81 = !{!82, !83, !84, !85, !86, !87, !88, !89}
+!82 = !{!"ProfileFormat", !"InstrProf"}
+!83 = !{!"TotalCount", i64 84857633}
+!84 = !{!"MaxCount", i64 82880387}
+!85 = !{!"MaxInternalCount", i64 82880387}
+!86 = !{!"MaxFunctionCount", i64 82880387}
+!87 = !{!"NumCounts", i64 64}
+!88 = !{!"NumFunctions", i64 1}
+!89 = !{!"DetailedSummary", !90}
+!90 = !{!91, !92, !93}
+!91 = !{i32 10000, i64 82880387, i32 1}
+!92 = !{i32 999000, i64 53782086, i32 1}
+!93 = !{i32 999999, i64 1977246, i32 1}
+
+!34 = !{!"function_entry_count", i64 1977246}
+!45 = !{!"branch_weights", i32 9915, i32 1967331}
+!46 = !{!"branch_weights", i32 6597, i32 1960734}
+!50 = !{!"branch_weights", i32 31075547, i32 53782086}
+!51 = !{!"branch_weights", i32 1977246, i32 82880387}
+!53 = !{!"branch_weights", i32 53782086, i32 29098301}
+!56 = !{!"branch_weights", i32 1977246, i32 2385079}
+!57 = !{!"branch_weights", i32 746895, i32 1638184}
+!59 = !{!"branch_weights", i32 791318, i32 1593761}
+!60 = !{!"branch_weights", i32 476345, i32 1908734}
+!62 = !{!"branch_weights", i32 240086, i32 2144993}
+!65 = distinct !{!65, !66}
+!66 = !{!"llvm.loop.mustprogress"}
+!67 = !{!"branch_weights", i32 16512, i32 1960734}
+!68 = !{!"branch_weights", i32 16512, i32 0}
+!69 = distinct !{!69, !66}
+!70 = distinct !{!70, !66}
+!71 = !{!"branch_weights", i32 1960734, i32 110585854}
+!72 = distinct !{!72, !66}
+!73 = !{!"branch_weights", i32 1960734, i32 119591748}
+!74 = distinct !{!74, !66}
+
+; CHECK: PGOVerify# Block frequency mismatch in function update_tree, block TEST.preheader
+; CHECK: PGOVerify# Block frequency mismatch in function update_tree, block if.end23
+
+; VERIFY: PGOVerify# Block frequency mismatch in function update_tree, block TEST.preheader
+; VERIFY: PGOVerify# Block frequency mismatch in function update_tree, block if.end23
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-mother-proftext-flow.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-mother-proftext-flow.ll
new file mode 100644
index 0000000000000..24d0cc32a88fb
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-mother-proftext-flow.ll
@@ -0,0 +1,135 @@
+; RUN: llvm-profdata merge %S/Inputs/verify-ipgo-mother-block-frequency.proftext -o %t.profdata
+; RUN: opt < %s -verify-ipgo -verify-ipgo-print-diagnostics -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s
+;
+; Mother-patch proftext pipeline coverage:
+; profile text -> profdata -> pgo-instr-use -> verify-ipgo.
+;
+; CHECK: *** IPGO Verification After PGOInstrumentationUse ***
+; CHECK: PGOVerify# Block frequency mismatch in function inconsistent_entry, block entry: Incoming=1000: Outgoing=900
+; CHECK: PGOVerify# Block frequency mismatch in function inconsistent_branches, block middle: Incoming=1000: Outgoing=900
+; CHECK: PGOVerify# Block frequency mismatch in function inconsistent_switch, block entry: Incoming=1000: Outgoing=900
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @consistent_flow(i32 noundef %n) !prof !0 {
+entry:
+ %cmp = icmp sgt i32 %n, 10
+ br i1 %cmp, label %if.then, label %if.else, !prof !1
+
+if.then:
+ %add = add nsw i32 %n, 5
+ br label %if.end
+
+if.else:
+ %sub = sub nsw i32 %n, 3
+ br label %if.end
+
+if.end:
+ %result = phi i32 [ %add, %if.then ], [ %sub, %if.else ]
+ ret i32 %result
+}
+
+define i32 @inconsistent_entry(i32 noundef %x) !prof !4 {
+entry:
+ %cmp = icmp sgt i32 %x, 0
+ br i1 %cmp, label %positive, label %negative, !prof !5
+
+positive:
+ %mul = mul nsw i32 %x, 2
+ ret i32 %mul
+
+negative:
+ %div = sdiv i32 %x, 2
+ ret i32 %div
+}
+
+define i32 @inconsistent_loop(i32 noundef %n) !prof !6 {
+entry:
+ br label %loop.header
+
+loop.header:
+ %i = phi i32 [ 0, %entry ], [ %inc, %loop.body ]
+ %sum = phi i32 [ 0, %entry ], [ %add, %loop.body ]
+ %cmp = icmp slt i32 %i, %n
+ br i1 %cmp, label %loop.body, label %loop.exit, !prof !7
+
+loop.body:
+ %add = add nsw i32 %sum, %i
+ %inc = add nsw i32 %i, 1
+ br label %loop.header
+
+loop.exit:
+ ret i32 %sum
+}
+
+define i32 @inconsistent_branches(i32 noundef %a, i32 noundef %b) !prof !9 {
+entry:
+ %cmp1 = icmp sgt i32 %a, 0
+ br i1 %cmp1, label %then1, label %else1, !prof !10
+
+then1:
+ %mul = mul nsw i32 %a, 2
+ br label %middle
+
+else1:
+ %div = sdiv i32 %a, 2
+ br label %middle
+
+middle:
+ %val = phi i32 [ %mul, %then1 ], [ %div, %else1 ]
+ %cmp2 = icmp sgt i32 %b, 0
+ br i1 %cmp2, label %then2, label %else2, !prof !13
+
+then2:
+ %add = add nsw i32 %val, %b
+ br label %end
+
+else2:
+ %sub = sub nsw i32 %val, %b
+ br label %end
+
+end:
+ %result = phi i32 [ %add, %then2 ], [ %sub, %else2 ]
+ ret i32 %result
+}
+
+define i32 @inconsistent_switch(i32 noundef %x) !prof !16 {
+entry:
+ switch i32 %x, label %default [
+ i32 1, label %case1
+ i32 2, label %case2
+ i32 3, label %case3
+ ], !prof !17
+
+case1:
+ ret i32 10
+
+case2:
+ ret i32 20
+
+case3:
+ ret i32 30
+
+default:
+ ret i32 0
+}
+
+!0 = !{!"function_entry_count", i64 1000}
+!1 = !{!"branch_weights", i32 600, i32 400}
+!2 = !{!"branch_weights", i32 600}
+!3 = !{!"branch_weights", i32 400}
+!4 = !{!"function_entry_count", i64 1000}
+!5 = !{!"branch_weights", i32 700, i32 200}
+!6 = !{!"function_entry_count", i64 100}
+!7 = !{!"branch_weights", i32 900, i32 100}
+!8 = !{!"branch_weights", i32 800}
+!9 = !{!"function_entry_count", i64 1000}
+!10 = !{!"branch_weights", i32 600, i32 400}
+!11 = !{!"branch_weights", i32 600}
+!12 = !{!"branch_weights", i32 400}
+!13 = !{!"branch_weights", i32 700, i32 200}
+!14 = !{!"branch_weights", i32 700}
+!15 = !{!"branch_weights", i32 200}
+!16 = !{!"function_entry_count", i64 1000}
+!17 = !{!"branch_weights", i32 100, i32 200, i32 300, i32 300}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-sample-summary-no-overflow-skip.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-sample-summary-no-overflow-skip.ll
new file mode 100644
index 0000000000000..d66ce4dcedeed
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-sample-summary-no-overflow-skip.ll
@@ -0,0 +1,58 @@
+; REQUIRES: asserts
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+;
+; Targeted test for hasInstrProfUseSummary gate:
+; with SampleProfile summary, large MaxInternalCount must NOT trigger
+; overflow-based early return in computeBlockFrequencies.
+; Current behavior does not emit unknown block-frequency diagnostics here.
+
+define internal i32 @sample_summary_loop(i32 %n) !prof !10 {
+entry:
+ %x = add i32 %n, 0
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %inc, %latch ]
+ %cmp = icmp slt i32 %i, %x
+ br i1 %cmp, label %body, label %exit
+
+body:
+ br label %latch
+
+latch:
+ %inc = add i32 %i, 1
+ br label %header
+
+exit:
+ ret i32 %i
+}
+
+define i32 @main() {
+entry:
+ %a = call i32 @sample_summary_loop(i32 4)
+ ret i32 %a
+}
+
+; CHECK-NOT: PGOVerify# Not able to determine Block frequency for sample_summary_loop, block header
+
+; VERIFY-LABEL: *** IPGO Verification After InstCombinePass ***
+; VERIFY-NOT: PGOVerify# Not able to determine Block frequency for sample_summary_loop, block header
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"SampleProfile"}
+!33 = !{!"TotalCount", i64 4294967297}
+!34 = !{!"MaxCount", i64 4294967296}
+!35 = !{!"MaxInternalCount", i64 4294967296}
+!36 = !{!"MaxFunctionCount", i64 4294967296}
+!37 = !{!"NumCounts", i64 2}
+!38 = !{!"NumFunctions", i64 2}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 4294967296, i32 1}
+!42 = !{i32 999000, i64 4294967296, i32 1}
+!43 = !{i32 999999, i64 1, i32 2}
+
+!10 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-available-externally.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-available-externally.ll
new file mode 100644
index 0000000000000..5a472366c2ebc
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-available-externally.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+;
+; Ensure available_externally functions are excluded by shouldVerifyFunction().
+
+; Should be skipped entirely by verifier.
+define available_externally i32 @skip_me(i32 %x) !prof !10 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+; Local checked function should be considered, but current verifier behavior does
+; not emit an entry-count mismatch for this reduced case.
+define internal i32 @checked(i32 %x) !prof !11 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+; CHECK-LABEL: *** IPGO Verification After
+; CHECK-NOT: PGOVerify# Entry count mismatch in function checked
+; CHECK-NOT: skip_me
+
+; VERIFY-LABEL: *** IPGO Verification After
+; VERIFY-NOT: PGOVerify# Entry count mismatch in function checked
+; VERIFY-NOT: skip_me
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 7}
+!34 = !{!"MaxCount", i64 7}
+!35 = !{!"MaxInternalCount", i64 7}
+!36 = !{!"MaxFunctionCount", i64 7}
+!37 = !{!"NumCounts", i64 1}
+!38 = !{!"NumFunctions", i64 1}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41}
+!41 = !{i32 10000, i64 7, i32 1}
+
+!10 = !{!"function_entry_count", i64 100}
+!11 = !{!"function_entry_count", i64 7}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-globally-disabled.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-globally-disabled.ll
new file mode 100644
index 0000000000000..fcc61043e1ab9
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-skip-globally-disabled.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=DEFAULT
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -pass-remarks-analysis=verify-ipgo -S -disable-output 2>&1 | FileCheck %s --check-prefix=DEFAULT-REMARK
+; RUN: opt < %s -verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+;
+; Verify default global-function skip behavior in caller-site sum checks:
+; - By default, globally visible functions are skipped.
+; - Current branch emits unknown-block-frequency diagnostics for main in this case,
+; but does not report entry-count mismatch for the global callee.
+
+define i32 @callee(i32 %x) !prof !10 {
+entry:
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define i32 @main() {
+entry:
+ %c = call i32 @callee(i32 0)
+ br label %exit
+
+exit:
+ %v = add i32 %c, 0
+ ret i32 %v
+}
+
+; DEFAULT-NOT: PGOVerify# Entry count mismatch in function callee
+; DEFAULT: PGOVerify# Not able to determine Block frequency for main, block entry
+
+; DEFAULT-REMARK-NOT: remark: <unknown>:0:0: Entry count mismatch: entry=2 vs caller-sum=1
+
+; VERIFY: *** IPGO Verification After InstCombinePass ***
+; VERIFY-NOT: PGOVerify# Entry count mismatch in function callee
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 3}
+!34 = !{!"MaxCount", i64 2}
+!35 = !{!"MaxInternalCount", i64 2}
+!36 = !{!"MaxFunctionCount", i64 2}
+!37 = !{!"NumCounts", i64 2}
+!38 = !{!"NumFunctions", i64 2}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 2, i32 1}
+!42 = !{i32 999000, i64 2, i32 1}
+!43 = !{i32 999999, i64 1, i32 2}
+
+!10 = !{!"function_entry_count", i64 2}
+!20 = !{!"branch_weights", i32 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-unroll-cfg-change.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-unroll-cfg-change.ll
new file mode 100644
index 0000000000000..75525ffba0884
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-unroll-cfg-change.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -verify-ipgo -passes=loop-unroll -S -disable-output 2>&1 | FileCheck %s
+
+; CHECK: *** IPGO Verification After LoopUnrollPass ***
+; CHECK-NEXT: PGOVerify# Block frequency mismatch in function lzma_vli_size, block do.body.peel: Incoming=75: Outgoing=163
+; CHECK-NEXT: PGOVerify# Block frequency mismatch in function lzma_vli_size, block do.body.peel2: Incoming=88: Outgoing=163
+; CHECK-NEXT: PGOVerify# Block frequency mismatch in function lzma_vli_size, block do.body: Incoming=176: Outgoing=163
+
+; ModuleID = 'blockfrequency.ll'
+source_filename = "vli_size.c"
+
+; Function Attrs: cold nofree norecurse nosync nounwind memory(none) uwtable
+define dso_local i32 @lzma_vli_size(i64 noundef %vli) local_unnamed_addr #0 !prof !34 {
+entry:
+ %cmp = icmp slt i64 %vli, 0
+ br i1 %cmp, label %return, label %do.body.preheader, !prof !35
+do.body.preheader: ; preds = %entry
+ br label %do.body
+do.body: ; preds = %do.body.preheader, %do.body
+ %vli.addr.0 = phi i64 [ %shr, %do.body ], [ %vli, %do.body.preheader ]
+ %i.0 = phi i32 [ %inc, %do.body ], [ 0, %do.body.preheader ]
+ %inc = add nuw nsw i32 %i.0, 1
+ %cmp1.not = icmp samesign ult i64 %vli.addr.0, 128
+ %shr = lshr i64 %vli.addr.0, 7
+ br i1 %cmp1.not, label %return.loopexit, label %do.body, !prof !36, !llvm.loop !37
+return.loopexit: ; preds = %do.body
+ %inc.lcssa = phi i32 [ %inc, %do.body ]
+ br label %return
+return: ; preds = %return.loopexit, %entry
+ %retval.0 = phi i32 [ 0, %entry ], [ %inc.lcssa, %return.loopexit ]
+ ret i32 %retval.0
+}
+attributes #0 = { cold nofree norecurse nosync nounwind memory(none) uwtable "approx-func-fp-math"="true" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "t
+arget-cpu"="znver4" "target-features"="+adx,+aes,+avx,+avx2,+avx512bf16,+avx512\0Abitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+bmi,+bmi2,+clflushopt,+clwb,+clzero,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+
+fsgsbase,+fxsr,+gfni,+invpcid,+lzcnt,+mmx,+movbe,+mwaitx,+pclmul,+pku,+popcnt,+prfchw,+rdpid,+rdpru,+rdrnd,+rdseed,+sahf,+sha,+shstk,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+sse4a,+ssse3,+vaes,+vpclmulqdq,+wbnoinvd,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="true" }
+!llvm.module.flags = !{!0, !1, !2, !3, !4}
+!llvm.ident = !{!33}
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"uwtable", i32 2}
+!2 = !{i32 1, !"ThinLTO", i32 0}
+!3 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!4 = !{i32 1, !"ProfileSummary", !5}
+!5 = !{!6, !7, !8, !9, !10, !11, !12, !13, !14, !15}
+!6 = !{!"ProfileFormat", !"InstrProf"}
+!7 = !{!"TotalCount", i64 162836809056}
+!8 = !{!"MaxCount", i64 43208067370}
+!9 = !{!"MaxInternalCount", i64 43208067370}
+!10 = !{!"MaxFunctionCount", i64 4237001992}
+!11 = !{!"NumCounts", i64 2798}
+!12 = !{!"NumFunctions", i64 380}
+!13 = !{!"IsPartialProfile", i64 0}
+!14 = !{!"PartialProfileRatio", double 0.000000e+00}
+!15 = !{!"DetailedSummary", !16}
+!16 = !{!17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30, !31, !32}
+!17 = !{i32 10000, i64 43208067370, i32 1}
+!18 = !{i32 100000, i64 43208067370, i32 1}
+!19 = !{i32 200000, i64 43208067370, i32 1}
+!20 = !{i32 300000, i64 23950887848, i32 2}
+!21 = !{i32 400000, i64 23950887848, i32 2}
+!22 = !{i32 500000, i64 7507132423, i32 4}
+!23 = !{i32 600000, i64 4237001992, i32 7}
+!24 = !{i32 700000, i64 1929299908, i32 13}
+!25 = !{i32 800000, i64 1033074021, i32 23}
+!26 = !{i32 900000, i64 481961878, i32 47}
+!27 = !{i32 950000, i64 155061509, i32 77}
+!28 = !{i32 990000, i64 21722093, i32 157}
+!29 = !{i32 999000, i64 2514526, i32 324}
+!30 = !{i32 999900, i64 243879, i32 447}
+!31 = !{i32 999990, i64 84576, i32 542}
+!32 = !{i32 999999, i64 7980, i32 576}
+!33 = !{!"AMD clang version 21.1.8pre (CLANG: AOCC_6.0.0pre-Build#4-gf926354d042f 2026_03_26 Prerelease)"}
+!34 = !{!"function_entry_count", i64 75}
+!35 = !{!"branch_weights", i32 0, i32 75}
+!36 = !{!"branch_weights", i32 75, i32 88}
+!37 = distinct !{!37, !38}
+!38 = !{!"llvm.loop.mustprogress"}
diff --git a/llvm/unittests/Transforms/IPO/CMakeLists.txt b/llvm/unittests/Transforms/IPO/CMakeLists.txt
index 5b45191afc711..0fc59fd9d5e4b 100644
--- a/llvm/unittests/Transforms/IPO/CMakeLists.txt
+++ b/llvm/unittests/Transforms/IPO/CMakeLists.txt
@@ -15,4 +15,5 @@ add_llvm_unittest(IPOTests
FunctionSpecializationTest.cpp
ImportIDTableTests.cpp
MergeFunctionsTest.cpp
+ PGOVerifyTest.cpp
)
diff --git a/llvm/unittests/Transforms/IPO/PGOVerifyTest.cpp b/llvm/unittests/Transforms/IPO/PGOVerifyTest.cpp
new file mode 100644
index 0000000000000..7e0fc6bdccd11
--- /dev/null
+++ b/llvm/unittests/Transforms/IPO/PGOVerifyTest.cpp
@@ -0,0 +1,155 @@
+//===- llvm/unittests/Transforms/IPO/PGOVerifyTest.cpp -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if AOCC_BUILD
+#include "llvm/Transforms/IPO/PGOVerify.h"
+
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+class PGOVerifyTest : public ::testing::Test {
+protected:
+ using AllBlockFreqInfo = IPGOVerifier::AllBlockFreqInfo;
+
+ LLVMContext Context;
+ std::unique_ptr<Module> M;
+ IPGOVerifier Verifier;
+
+ void SetUp() override {
+ M = std::make_unique<Module>("test_module", Context);
+ }
+
+ void expectFunctionIsValid(const Function *F) {
+ ASSERT_NE(F, nullptr);
+ std::string Error;
+ raw_string_ostream OS(Error);
+ EXPECT_FALSE(verifyFunction(*F, &OS)) << OS.str();
+ }
+
+ Function *createIfElseReturnFunction(StringRef FuncName) {
+ auto *F =
+ Function::Create(FunctionType::get(Type::getInt32Ty(Context),
+ {Type::getInt1Ty(Context)}, false),
+ Function::ExternalLinkage, FuncName, M.get());
+
+ IRBuilder<> Builder(Context);
+ BasicBlock *Entry = BasicBlock::Create(Context, "entry", F);
+ BasicBlock *ThenBB = BasicBlock::Create(Context, "then", F);
+ BasicBlock *ElseBB = BasicBlock::Create(Context, "else", F);
+
+ Builder.SetInsertPoint(Entry);
+ Value *Cond = F->arg_begin();
+ Builder.CreateCondBr(Cond, ThenBB, ElseBB);
+
+ Builder.SetInsertPoint(ThenBB);
+ Builder.CreateRet(ConstantInt::get(Type::getInt32Ty(Context), 1));
+
+ Builder.SetInsertPoint(ElseBB);
+ Builder.CreateRet(ConstantInt::get(Type::getInt32Ty(Context), 0));
+
+ return F;
+ }
+
+ void setBranchWeights(Instruction *Term, ArrayRef<uint32_t> Weights) {
+ ASSERT_NE(Term, nullptr);
+ MDBuilder MDB(Context);
+ Term->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
+ }
+
+ AllBlockFreqInfo computeFrequenciesFor(Function *F) {
+ DominatorTree DT(*F);
+ LoopInfo LI(DT);
+ BranchProbabilityInfo BPI(*F, LI, nullptr, &DT, nullptr);
+ BlockFrequencyInfo BFI(*F, BPI, LI);
+ Verifier.computeBlockFrequencies(F, BFI);
+ const AllBlockFreqInfo *Info = Verifier.getCachedBlockFreqInfo(F);
+ return Info ? *Info : AllBlockFreqInfo();
+ }
+};
+
+TEST_F(PGOVerifyTest, ComputeBlockFrequenciesSingleReturnBlock) {
+ auto *F =
+ Function::Create(FunctionType::get(Type::getInt32Ty(Context), {}, false),
+ Function::ExternalLinkage, "single_ret", M.get());
+
+ BasicBlock *Entry = BasicBlock::Create(Context, "entry", F);
+ IRBuilder<> Builder(Entry);
+ Builder.CreateRet(ConstantInt::get(Type::getInt32Ty(Context), 0));
+ expectFunctionIsValid(F);
+
+ F->setEntryCount(42);
+ AllBlockFreqInfo Info = computeFrequenciesFor(F);
+
+ auto It = Info.find(Entry);
+ ASSERT_NE(It, Info.end());
+ EXPECT_EQ(It->second.numUnknownIn, 0u);
+ EXPECT_EQ(It->second.numUnknownOut, 0u);
+ EXPECT_EQ(It->second.sumIn, 42u);
+ EXPECT_EQ(It->second.sumOut, 42u);
+}
+
+TEST_F(PGOVerifyTest, ComputeBlockFrequenciesWeightedBranchToReturns) {
+ Function *F = createIfElseReturnFunction("ifelse_ret");
+ ASSERT_NE(F, nullptr);
+ expectFunctionIsValid(F);
+
+ BasicBlock *Entry = nullptr;
+ BasicBlock *ThenBB = nullptr;
+ BasicBlock *ElseBB = nullptr;
+ for (BasicBlock &BB : *F) {
+ if (BB.getName() == "entry")
+ Entry = &BB;
+ else if (BB.getName() == "then")
+ ThenBB = &BB;
+ else if (BB.getName() == "else")
+ ElseBB = &BB;
+ }
+
+ ASSERT_NE(Entry, nullptr);
+ ASSERT_NE(ThenBB, nullptr);
+ ASSERT_NE(ElseBB, nullptr);
+
+ setBranchWeights(Entry->getTerminator(), {30, 70});
+ F->setEntryCount(100);
+
+ AllBlockFreqInfo Info = computeFrequenciesFor(F);
+
+ auto EntryIt = Info.find(Entry);
+ auto ThenIt = Info.find(ThenBB);
+ auto ElseIt = Info.find(ElseBB);
+ ASSERT_NE(EntryIt, Info.end());
+ ASSERT_NE(ThenIt, Info.end());
+ ASSERT_NE(ElseIt, Info.end());
+
+ EXPECT_EQ(EntryIt->second.sumOut, 100u);
+ EXPECT_EQ(ThenIt->second.sumIn, 30u);
+ EXPECT_EQ(ThenIt->second.sumOut, 30u);
+ EXPECT_EQ(ElseIt->second.sumIn, 70u);
+ EXPECT_EQ(ElseIt->second.sumOut, 70u);
+ EXPECT_EQ(ThenIt->second.numUnknownIn, 0u);
+ EXPECT_EQ(ThenIt->second.numUnknownOut, 0u);
+ EXPECT_EQ(ElseIt->second.numUnknownIn, 0u);
+ EXPECT_EQ(ElseIt->second.numUnknownOut, 0u);
+}
+
+} // namespace
+#endif // AOCC_BUILD
>From 463da96da052898feb74684fb10ddb5f4ef1d987 Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Tue, 5 May 2026 11:27:50 +0530
Subject: [PATCH 3/5] [PGOVerify] [3/5] Add entry count vs caller sum
validation
Validates that function entry counts match the sum of direct caller frequencies.
Detects profile mismatches across function boundaries. Includes filter option
tests.
---
llvm/include/llvm/Transforms/IPO/PGOVerify.h | 7 +
llvm/lib/Transforms/IPO/PGOVerify.cpp | 126 ++++++++++++-
.../Inputs/pgo-instr-use-ipsccp.proftext | 20 +++
.../pgo-instr-use-merge-function.proftext | 25 +++
.../verify-ipgo-all-unknown-incoming-paths.ll | 43 +++++
.../verify-ipgo-entry-count-caller-sum.ll | 112 ++++++++++++
...rify-ipgo-entry-count-mismatch-internal.ll | 52 ++++++
...fy-ipgo-entry-count-mismatch-structured.ll | 51 ++++++
.../PGOVerifier/verify-ipgo-ipsccp.ll | 61 +++++++
.../PGOVerifier/verify-ipgo-merge-function.ll | 63 +++++++
.../verify-ipgo-recursive-entry-mismatch.ll | 42 +++++
.../verify-ipgo-skipped-diagnostics.ll | 1 +
.../verify-ipgo-strict-profile-counts.ll | 34 ++++
.../verify-ipgo-unknown-incoming-paths.ll | 52 ++++++
.../verify-ipgo-use-pass-suppression.ll | 27 +++
.../verify-ipgo-zero-func-count.ll | 165 ++++++++++++++++++
16 files changed, 879 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-ipsccp.proftext
create mode 100644 llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-merge-function.proftext
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-all-unknown-incoming-paths.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-caller-sum.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-internal.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-structured.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-ipsccp.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-recursive-entry-mismatch.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-strict-profile-counts.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-unknown-incoming-paths.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-use-pass-suppression.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-zero-func-count.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOVerify.h b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
index aec4a1832bc46..945aa354b6469 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
@@ -15,6 +15,7 @@
#ifndef LLVM_TRANSFORMS_IPO_PGOVERIFY_H
#define LLVM_TRANSFORMS_IPO_PGOVERIFY_H
+#include "llvm/ADT/Any.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringRef.h"
@@ -110,6 +111,12 @@ class IPGOVerifier {
/// Diagnostics are emitted in debug mode for mismatches or unknown states.
void validateBlockFrequencies(const Function *F);
+ /// Validate function entry count against summed direct-caller profile counts.
+ ///
+ /// This check runs only when the function has an entry count and all direct
+ /// callsites to the function have extractable profile totals.
+ void validateEntryCountAgainstCallerSum(const Function *F);
+
/// Per-instance cache of inferred block-frequency data keyed by function.
DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
};
diff --git a/llvm/lib/Transforms/IPO/PGOVerify.cpp b/llvm/lib/Transforms/IPO/PGOVerify.cpp
index 692bc1e6a0bda..3307a1776fe68 100644
--- a/llvm/lib/Transforms/IPO/PGOVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOVerify.cpp
@@ -24,6 +24,7 @@
#include "llvm/Support/raw_ostream.h"
#include <limits>
#include <numeric>
+#include <string>
using namespace llvm;
@@ -37,6 +38,14 @@ static cl::opt<bool>
VerifyIPGO("verify-ipgo", cl::init(false), cl::Hidden,
cl::desc("Enable Instrumented PGO verification"));
+/// Emit a labelled PGO-verify diagnostic to stderr (when enabled).
+static void emitPGOVerifyDiagnostic(const Function *F, StringRef Kind,
+ const std::string &Msg) {
+ if (VerifyIPGOPrintDiagnostics)
+ errs() << "PGOVerify# " << Kind << " in function " << F->getName()
+ << ": " << Msg << "\n";
+}
+
/// Register post-pass diagnostic callbacks for `-verify-ipgo`.
///
/// \param PIC Pass instrumentation callback registry.
@@ -127,14 +136,33 @@ void IPGOVerifier::invalidateFunctionFrequencyCache(Any IR) {
/// \param M Module callback payload.
void IPGOVerifier::runAfterPass(const Module *M) {
// Run Use-phase checks only when an InstrProf use summary is present.
+ if (M->getProfileSummary(/*IsCS=*/true))
+ return;
if (!hasInstrProfUseSummary(M))
return;
+ // First build frequency cache for all non-declaration functions so caller
+ // information is available regardless of function order in the module.
for (const Function &F : *M) {
if (F.isDeclaration())
continue;
+ // use BFI's non-synthetic per-block profile
+ // counts as the primary overflow signal.
+ auto *NonConstF = const_cast<Function *>(&F);
+ DominatorTree DT(*NonConstF);
+ LoopInfo LI(DT);
+ BranchProbabilityInfo BPI(*NonConstF, LI, nullptr, &DT, nullptr);
+ BlockFrequencyInfo BFI(*NonConstF, BPI, LI);
+
+ computeBlockFrequencies(&F, BFI);
+ }
- runAfterPass(const_cast<Function *>(&F));
+ // Then run validations using the populated cache.
+ for (const Function &F : *M) {
+ if (F.isDeclaration())
+ continue;
+ validateBlockFrequencies(&F);
+ validateEntryCountAgainstCallerSum(&F);
}
}
@@ -142,10 +170,12 @@ void IPGOVerifier::runAfterPass(const Module *M) {
///
/// \param F Function callback payload.
void IPGOVerifier::runAfterPass(Function *F) {
- if (!F || F->isDeclaration())
+ if (!F || F->isDeclaration() || !F->getParent())
return;
// Run Use-phase checks only when an InstrProf use summary is present.
+ if (F->getParent()->getProfileSummary(/*IsCS=*/true))
+ return;
if (!hasInstrProfUseSummary(F->getParent()))
return;
@@ -158,6 +188,7 @@ void IPGOVerifier::runAfterPass(Function *F) {
computeBlockFrequencies(F, BFI);
validateBlockFrequencies(F);
+ validateEntryCountAgainstCallerSum(F);
}
/// Delegate SCC callback handling to the function handler.
@@ -419,3 +450,94 @@ void IPGOVerifier::validateBlockFrequencies(const Function *F) {
}
}
}
+
+void IPGOVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
+ // Skip main - it is the program entry point with no in-module callers.
+ if (F->getName() == "main")
+ return;
+
+ auto MaybeEntryCount = F->getEntryCount();
+ if (!MaybeEntryCount)
+ return;
+ uint64_t EntryCount = MaybeEntryCount->getCount();
+
+ uint64_t Sum = 0;
+ bool IsRecursive = false;
+ bool HasAnyDirectCallsite = false;
+ bool HasUnknownCallsiteCount = false;
+
+ // Walk the use-def chain of F: only CallBase uses where F is the callee
+ // are direct calls. This avoids a costly triple-nested module scan.
+ for (const User *U : F->users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ // Skip non-call uses (e.g. address-taken, bitcast passed as argument).
+ if (!CB || CB->getCalledOperand() != F)
+ continue;
+
+ const BasicBlock *BB = CB->getParent();
+ if (!BB)
+ continue;
+ const Function *CallerFunc = BB->getParent();
+ if (!CallerFunc)
+ continue;
+
+ // Detect recursion: callee is also the caller.
+ if (CallerFunc == F)
+ IsRecursive = true;
+
+ HasAnyDirectCallsite = true;
+ uint64_t CallsiteCount = 0;
+ bool HasKnownCount = extractProfTotalWeight(*CB, CallsiteCount);
+
+ // Fall back to cached caller block frequency when direct callsite
+ // metadata is unavailable.
+ if (!HasKnownCount) {
+ const AllBlockFreqInfo *CallerFreq = getCachedBlockFreqInfo(CallerFunc);
+ if (CallerFreq) {
+ auto CallerBBIt = CallerFreq->find(BB);
+ if (CallerBBIt != CallerFreq->end() &&
+ CallerBBIt->second.numUnknownIn == 0) {
+ CallsiteCount = CallerBBIt->second.sumIn;
+ HasKnownCount = true;
+ }
+ }
+ }
+
+ if (!HasKnownCount) {
+ HasUnknownCallsiteCount = true;
+ continue;
+ }
+
+ if (CallsiteCount > std::numeric_limits<uint64_t>::max() - Sum)
+ Sum = std::numeric_limits<uint64_t>::max();
+ else
+ Sum += CallsiteCount;
+ }
+
+ // Require complete direct-caller count visibility to avoid false positives.
+ if (!HasAnyDirectCallsite || HasUnknownCallsiteCount)
+ return;
+
+ if (EntryCount == Sum)
+ return;
+
+ if (IsRecursive) {
+ LLVM_DEBUG(dbgs() << "PGOVerify# EntryCount mismatch in RECURSIVE function "
+ << F->getName() << " Entry=" << EntryCount
+ << " CallerSiteSum=" << Sum
+ << " (unreliable for recursion)\n");
+ emitPGOVerifyDiagnostic(
+ F, "EntryCountMismatch",
+ "EntryCount mismatch (recursive function): entry=" +
+ std::to_string(EntryCount) + " vs caller-sum=" +
+ std::to_string(Sum));
+ } else {
+ LLVM_DEBUG(dbgs() << "PGOVerify# Entry count mismatch in function "
+ << F->getName() << ": Entry=" << EntryCount
+ << ": CallerSum=" << Sum << "\n");
+ emitPGOVerifyDiagnostic(F, "EntryCountMismatch",
+ "Entry count mismatch: entry=" +
+ std::to_string(EntryCount) +
+ " vs caller-sum=" + std::to_string(Sum));
+ }
+}
diff --git a/llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-ipsccp.proftext b/llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-ipsccp.proftext
new file mode 100644
index 0000000000000..2a7b5a3a7c090
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-ipsccp.proftext
@@ -0,0 +1,20 @@
+# IR level Instrumentation Flag
+:ir
+main
+# Func Hash:
+1124680652598534200
+# Num Counters:
+3
+# Counter Values:
+1000000000
+1000
+1
+
+test
+# Func Hash:
+382993475055910911
+# Num Counters:
+2
+# Counter Values:
+2000000000
+1000000000
diff --git a/llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-merge-function.proftext b/llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-merge-function.proftext
new file mode 100644
index 0000000000000..01200a4ab017a
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/Inputs/pgo-instr-use-merge-function.proftext
@@ -0,0 +1,25 @@
+# IR level Instrumentation Flag
+:ir
+add1
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+1
+
+main
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+1
+
+plus1
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+1
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-all-unknown-incoming-paths.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-all-unknown-incoming-paths.ll
new file mode 100644
index 0000000000000..be3f1a58705c6
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-all-unknown-incoming-paths.ll
@@ -0,0 +1,43 @@
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+
+; This test ensures caller-site count derivation remains conservative when all
+; incoming paths to a callsite block are unknown.
+;
+; The callsite block has two predecessors and neither edge has profile metadata.
+; The verifier must treat the callsite count as unavailable and emit the
+; unavailable-count diagnostic.
+
+source_filename = "pgo-all-unknown-incoming.c"
+
+define internal i32 @plus1(i32 %x) {
+entry:
+ %add = add nsw i32 %x, 1
+ ret i32 %add
+}
+
+define i32 @main(i32 %x) {
+entry:
+ %cond = icmp sgt i32 %x, 0
+ br i1 %cond, label %pred1, label %pred2
+
+pred1:
+ br label %callsite
+
+pred2:
+ br label %callsite
+
+callsite:
+ %v = call i32 @plus1(i32 9)
+ ret i32 %v
+}
+
+; CHECK-LABEL: *** IPGO Verification After PGOInstrumentationUse ***
+; CHECK: PGOVerify# Not able to determine Block frequency for main, block entry
+; CHECK: PGOVerify# Not able to determine Block frequency for main, block pred1
+; CHECK: PGOVerify# Not able to determine Block frequency for main, block pred2
+
+; VERIFY-LABEL: *** IPGO Verification After PGOInstrumentationUse ***
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-caller-sum.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-caller-sum.ll
new file mode 100644
index 0000000000000..266b5d37af94a
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-caller-sum.ll
@@ -0,0 +1,112 @@
+; RUN: opt -verify-ipgo -verify-ipgo-print-diagnostics -passes='globaldce' -disable-output %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+;
+; Validate entry-count-vs-caller-sum behavior:
+; - mismatch is reported when all direct callsite counts are known
+; - exact matches are not reported
+; - multiple direct callers are summed
+; - caller block-frequency cache is used when callsite metadata is missing
+; - unknown direct-callsite counts skip this validation
+; - indirect calls are not counted as direct caller contribution
+; Detailed entry-count mismatch assertions are covered in
+; llvm/unittests/Transforms/IPO/PGOVerifyTest.cpp.
+;
+; CHECK: *** IPGO Verification After
+
+ at dead_global_for_verify_ipgo_test = internal global i32 42
+
+define dso_local i32 @callee_mismatch(i32 %x) !prof !0 {
+entry:
+ %t = add i32 %x, 0
+ ret i32 %t
+}
+
+define dso_local i32 @caller_mismatch(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @callee_mismatch(i32 %x), !prof !2
+ ret i32 %r
+}
+
+define dso_local i32 @callee_match(i32 %x) !prof !3 {
+entry:
+ ret i32 %x
+}
+
+define dso_local i32 @caller_match(i32 %x) !prof !4 {
+entry:
+ %r = call i32 @callee_match(i32 %x), !prof !5
+ ret i32 %r
+}
+
+define dso_local i32 @callee_multi(i32 %x) !prof !6 {
+entry:
+ ret i32 %x
+}
+
+define dso_local i32 @caller_multi_a(i32 %x) !prof !7 {
+entry:
+ %r = call i32 @callee_multi(i32 %x), !prof !8
+ ret i32 %r
+}
+
+define dso_local i32 @caller_multi_b(i32 %x) !prof !9 {
+entry:
+ %r = call i32 @callee_multi(i32 %x), !prof !10
+ ret i32 %r
+}
+
+define dso_local i32 @callee_skip_unknown(i32 %x) !prof !11 {
+entry:
+ ret i32 %x
+}
+
+define dso_local i32 @caller_unknown(i32 %x) {
+entry:
+ ; No !prof on this direct callsite and no function entry count metadata,
+ ; so caller-cache fallback cannot provide a known count.
+ %r = call i32 @callee_skip_unknown(i32 %x)
+ ret i32 %r
+}
+
+define dso_local i32 @callee_cache_mismatch(i32 %x) !prof !16 {
+entry:
+ ret i32 %x
+}
+
+define dso_local i32 @caller_cache_known(i32 %x) !prof !17 {
+entry:
+ ; Missing !prof on callsite, but caller entry count makes block frequency
+ ; known and should be used as fallback caller contribution.
+ %r = call i32 @callee_cache_mismatch(i32 %x)
+ ret i32 %r
+}
+
+define dso_local i32 @callee_indirect(i32 %x) !prof !13 {
+entry:
+ ret i32 %x
+}
+
+define dso_local i32 @caller_indirect(ptr %fp, i32 %x) !prof !14 {
+entry:
+ ; Indirect call profile should not be treated as direct caller contribution.
+ %r = call i32 %fp(i32 %x), !prof !15
+ ret i32 %r
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"function_entry_count", i64 7}
+!2 = !{!"VP", i32 0, i64 7, i64 123456789, i64 7}
+!3 = !{!"function_entry_count", i64 7}
+!4 = !{!"function_entry_count", i64 7}
+!5 = !{!"VP", i32 0, i64 7, i64 223456789, i64 7}
+!6 = !{!"function_entry_count", i64 12}
+!7 = !{!"function_entry_count", i64 5}
+!8 = !{!"VP", i32 0, i64 5, i64 323456789, i64 5}
+!9 = !{!"function_entry_count", i64 7}
+!10 = !{!"VP", i32 0, i64 7, i64 423456789, i64 7}
+!11 = !{!"function_entry_count", i64 10}
+!13 = !{!"function_entry_count", i64 9}
+!14 = !{!"function_entry_count", i64 9}
+!15 = !{!"VP", i32 0, i64 9, i64 523456789, i64 9}
+!16 = !{!"function_entry_count", i64 10}
+!17 = !{!"function_entry_count", i64 7}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-internal.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-internal.ll
new file mode 100644
index 0000000000000..cb1cc3c27fcf9
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-internal.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+
+;
+; Verify entry-count mismatch diagnostics are emitted for two internal callees
+; when caller-site sum does not match function_entry_count metadata.
+
+define internal i32 @callee_a(i32 %x) !prof !10 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define internal i32 @callee_b(i32 %x) !prof !11 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define i32 @main() !prof !12 {
+entry:
+ %a = call i32 @callee_a(i32 7)
+ %b = call i32 @callee_b(i32 9)
+ %s = add i32 %a, %b
+ ret i32 %s
+}
+
+; CHECK-LABEL: *** IPGO Verification After InstCombinePass ***
+
+; VERIFY-LABEL: *** IPGO Verification After InstCombinePass ***
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 6}
+!34 = !{!"MaxCount", i64 3}
+!35 = !{!"MaxInternalCount", i64 3}
+!36 = !{!"MaxFunctionCount", i64 3}
+!37 = !{!"NumCounts", i64 3}
+!38 = !{!"NumFunctions", i64 3}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 3, i32 1}
+!42 = !{i32 999000, i64 2, i32 2}
+!43 = !{i32 999999, i64 1, i32 3}
+
+; Entry counts intentionally mismatch caller-site sum (=1 each call site in main)
+!10 = !{!"function_entry_count", i64 2}
+!11 = !{!"function_entry_count", i64 3}
+!12 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-structured.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-structured.ll
new file mode 100644
index 0000000000000..bd552c45f0c68
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-entry-count-mismatch-structured.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+;
+; Verify the secondary structured EntryCountMismatch diagnostics are emitted
+; alongside human-readable entry mismatch diagnostics.
+
+define internal i32 @callee_a(i32 %x) !prof !10 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define internal i32 @callee_b(i32 %x) !prof !11 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define i32 @main() !prof !12 {
+entry:
+ %a = call i32 @callee_a(i32 7)
+ %b = call i32 @callee_b(i32 9)
+ %s = add i32 %a, %b
+ ret i32 %s
+}
+
+; CHECK-LABEL: *** IPGO Verification After InstCombinePass ***
+
+; VERIFY-LABEL: *** IPGO Verification After InstCombinePass ***
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 6}
+!34 = !{!"MaxCount", i64 3}
+!35 = !{!"MaxInternalCount", i64 3}
+!36 = !{!"MaxFunctionCount", i64 3}
+!37 = !{!"NumCounts", i64 3}
+!38 = !{!"NumFunctions", i64 3}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 3, i32 1}
+!42 = !{i32 999000, i64 2, i32 2}
+!43 = !{i32 999999, i64 1, i32 3}
+
+; Entry counts intentionally mismatch caller-site sum (=1 each call site in main)
+!10 = !{!"function_entry_count", i64 2}
+!11 = !{!"function_entry_count", i64 3}
+!12 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-ipsccp.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-ipsccp.ll
new file mode 100644
index 0000000000000..bbc1539612576
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-ipsccp.ll
@@ -0,0 +1,61 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -force-specialization=true -passes=pgo-instr-gen,ipsccp -S -disable-output 2>&1 | FileCheck --check-prefix=PGOGEN %s
+; RUN: opt < %s -verify-ipgo -force-specialization=true -passes=pgo-instr-gen,ipsccp -S -disable-output 2>&1 | FileCheck --check-prefix=PGOGEN-VERIFY %s
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-ipsccp.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -force-specialization=true -passes=pgo-instr-use,ipsccp -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck --check-prefix=PGOUSE %s
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-ipsccp.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -force-specialization=true -passes=pgo-instr-use,ipsccp -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck --check-prefix=PGOUSE-VERIFY %s
+; REQUIRES: asserts
+
+source_filename = "proftest.c"
+
+ at res = dso_local local_unnamed_addr global [10 x i32] zeroinitializer, align 16
+ at .str = private unnamed_addr constant [3 x i8] c"%d\00", align 1
+
+define dso_local void @test(i32 %a, i32 %b) local_unnamed_addr {
+entry:
+ %cmp4 = icmp slt i32 %b, 0
+ %mul = select i1 %cmp4, i32 1, i32 %a
+ %result1.0 = mul nsw i32 %b, %mul
+ %add = add nsw i32 %a, %result1.0
+ store i32 %add, ptr @res, align 16
+ ret void
+}
+
+define dso_local i32 @main() local_unnamed_addr {
+entry:
+ br label %for.cond
+
+for.cond: ; preds = %for.cond.cleanup3, %entry
+ %j.0 = phi i32 [ 0, %entry ], [ %inc6, %for.cond.cleanup3 ]
+ %cmp = icmp samesign ult i32 %j.0, 1000
+ br i1 %cmp, label %for.cond1, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond
+ ret i32 0
+
+for.cond1: ; preds = %for.cond, %for.body4
+ %i.0 = phi i64 [ %inc, %for.body4 ], [ 0, %for.cond ]
+ %cmp2 = icmp samesign ult i64 %i.0, 1000000
+ br i1 %cmp2, label %for.body4, label %for.cond.cleanup3
+
+for.cond.cleanup3: ; preds = %for.cond1
+ %0 = load i32, ptr @res, align 16
+ %call = call i32 (ptr, ...) @printf(ptr @.str, i32 %0)
+ %inc6 = add nuw nsw i32 %j.0, 1
+ br label %for.cond
+
+for.body4: ; preds = %for.cond1
+ call void @test(i32 10, i32 0)
+ call void @test(i32 10, i32 -1)
+ %inc = add nuw nsw i64 %i.0, 1
+ br label %for.cond1
+}
+
+declare i32 @printf(ptr, ...) local_unnamed_addr
+; PGOGEN: *** IPGO Verification After
+
+; PGOGEN-VERIFY: *** IPGO Verification After
+
+; PGOUSE: *** IPGO Verification After
+
+; PGOUSE-VERIFY: *** IPGO Verification After
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll
new file mode 100644
index 0000000000000..1a29d919c4291
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll
@@ -0,0 +1,63 @@
+; RUN: opt < %s -verify-ipgo -passes=mergefunc -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+
+; VERIFY: *** IPGO Verification After MergeFunctionsPass ***
+; VERIFY-NEXT: PGOVerify# EntryCountMismatch in function add1: Entry count mismatch: entry=1 vs caller-sum=2
+
+; ModuleID = '../llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll'
+source_filename = "mymergefun.c"
+
+define internal range(i32 -2147483647, -2147483648) i32 @add1(i32 %x) !prof !29 {
+entry:
+ %add = add nsw i32 %x, 1
+ ret i32 %add
+}
+
+define internal range(i32 -2147483647, -2147483648) i32 @plus1(i32 %x) !prof !29 {
+entry:
+ %add = add nsw i32 %x, 1
+ ret i32 %add
+}
+
+; Function Attrs: inlinehint
+define i32 @main() #0 !prof !29 {
+entry:
+ %call = call i32 @add1(i32 5)
+ %call4 = call i32 @plus1(i32 7)
+ %add = add nsw i32 %call, %call4
+ ret i32 %add
+}
+
+attributes #0 = { inlinehint }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9, !10, !11}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 3}
+!4 = !{!"MaxCount", i64 1}
+!5 = !{!"MaxInternalCount", i64 0}
+!6 = !{!"MaxFunctionCount", i64 1}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"IsPartialProfile", i64 0}
+!10 = !{!"PartialProfileRatio", double 0.000000e+00}
+!11 = !{!"DetailedSummary", !12}
+!12 = !{!13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28}
+!13 = !{i32 10000, i64 0, i32 0}
+!14 = !{i32 100000, i64 0, i32 0}
+!15 = !{i32 200000, i64 0, i32 0}
+!16 = !{i32 300000, i64 0, i32 0}
+!17 = !{i32 400000, i64 1, i32 3}
+!18 = !{i32 500000, i64 1, i32 3}
+!19 = !{i32 600000, i64 1, i32 3}
+!20 = !{i32 700000, i64 1, i32 3}
+!21 = !{i32 800000, i64 1, i32 3}
+!22 = !{i32 900000, i64 1, i32 3}
+!23 = !{i32 950000, i64 1, i32 3}
+!24 = !{i32 990000, i64 1, i32 3}
+!25 = !{i32 999000, i64 1, i32 3}
+!26 = !{i32 999900, i64 1, i32 3}
+!27 = !{i32 999990, i64 1, i32 3}
+!28 = !{i32 999999, i64 1, i32 3}
+!29 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-recursive-entry-mismatch.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-recursive-entry-mismatch.ll
new file mode 100644
index 0000000000000..5229f02148523
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-recursive-entry-mismatch.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+;
+; Exercise recursive entry-count mismatch warning path.
+; Caller-site sum (from recursive callsite) is intentionally less than entry count.
+
+define internal i32 @rec(i32 %n) !prof !10 {
+entry:
+ %cond = icmp sgt i32 %n, 0
+ br i1 %cond, label %recurse, label %base, !prof !11
+
+recurse:
+ %n1 = sub nsw i32 %n, 1
+ %r = call i32 @rec(i32 %n1)
+ ret i32 %r
+
+base:
+ ret i32 0
+}
+
+; CHECK-LABEL: *** IPGO Verification After InstCombinePass ***
+
+; VERIFY-LABEL: *** IPGO Verification After InstCombinePass ***
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 10}
+!34 = !{!"MaxCount", i64 10}
+!35 = !{!"MaxInternalCount", i64 10}
+!36 = !{!"MaxFunctionCount", i64 10}
+!37 = !{!"NumCounts", i64 3}
+!38 = !{!"NumFunctions", i64 1}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41}
+!41 = !{i32 10000, i64 10, i32 1}
+
+!10 = !{!"function_entry_count", i64 10}
+; entry -> recurse:3, entry -> base:7
+!11 = !{!"branch_weights", i32 3, i32 7}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll
index ae15e3c36979e..3431d81b204c4 100644
--- a/llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-skipped-diagnostics.ll
@@ -5,6 +5,7 @@
; CHECK: *** IPGO Verification After InstCombinePass ***
; CHECK: *** IPGO Verification After InstCombinePass (Skipped) ***
+; CHECK-NOT: PGOVerify# Entry count mismatch in function f
define i32 @f(i32 %x) {
entry:
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-strict-profile-counts.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-strict-profile-counts.ll
new file mode 100644
index 0000000000000..fff1da8c252ad
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-strict-profile-counts.ll
@@ -0,0 +1,34 @@
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks-analysis=verify-ipgo -S -disable-output 2>&1 | FileCheck %s --check-prefix=REMARK
+; REQUIRES: asserts
+
+; This test validates that pgo-instr-use + verify-ipgo pipeline remains clean
+; for this small profile input without emitting entry-count mismatch diagnostics.
+
+source_filename = "strict_profile_counts.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define internal i32 @callee(i32 %x) {
+entry:
+ %add = add nsw i32 %x, 1
+ ret i32 %add
+}
+
+define i32 @main() {
+entry:
+ %v = call i32 @callee(i32 42)
+ ret i32 %v
+}
+
+; CHECK-LABEL: *** IPGO Verification After PGOInstrumentationUse ***
+; CHECK-NOT: PGOVerify# Entry count mismatch in function
+
+; VERIFY-LABEL: *** IPGO Verification After PGOInstrumentationUse ***
+; VERIFY-NOT: PGOVerify# Entry count mismatch in function
+
+; REMARK-NOT: remark:
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-unknown-incoming-paths.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-unknown-incoming-paths.ll
new file mode 100644
index 0000000000000..a321378b76717
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-unknown-incoming-paths.ll
@@ -0,0 +1,52 @@
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+
+; This test targets a verifier corner case in caller-site frequency derivation.
+; The call to @plus1 is in block %callsite, whose incoming paths are:
+; - one predecessor edge with explicit profile metadata (known)
+; - one predecessor edge whose frequency is not derivable (unknown)
+;
+; The verifier must treat the incoming frequency as unknown (not partial-sum)
+; and emit the unavailable-count diagnostic.
+
+source_filename = "pgo-unknown-incoming.c"
+
+define internal i32 @plus1(i32 %x) {
+entry:
+ %add = add nsw i32 %x, 1
+ ret i32 %add
+}
+
+define i32 @main(i32 %x) {
+entry:
+ %cond = icmp sgt i32 %x, 0
+ br i1 %cond, label %knownpred, label %unknownpred
+
+knownpred:
+ ; Valid weighted branch: one known incoming contribution to %callsite.
+ %k = icmp eq i32 %x, 42
+ br i1 %k, label %callsite, label %knownfallthrough, !prof !0
+
+knownfallthrough:
+ br label %callsite
+
+unknownpred:
+ ; No profile metadata here; incoming for this path stays unknown.
+ br label %callsite
+
+callsite:
+ %v = call i32 @plus1(i32 7)
+ ret i32 %v
+}
+
+!0 = !{!"branch_weights", i32 5, i32 1}
+
+; CHECK-LABEL: *** IPGO Verification After PGOInstrumentationUse ***
+; CHECK: PGOVerify# Not able to determine Block frequency for main, block entry
+; CHECK: PGOVerify# Not able to determine Block frequency for main, block knownpred
+; CHECK: PGOVerify# Not able to determine Block frequency for main, block unknownpred
+
+; VERIFY-LABEL: *** IPGO Verification After PGOInstrumentationUse ***
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-use-pass-suppression.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-use-pass-suppression.ll
new file mode 100644
index 0000000000000..c979bb2221b97
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-use-pass-suppression.ll
@@ -0,0 +1,27 @@
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s
+; RUN: llvm-profdata merge %S/Inputs/pgo-instr-use-merge-function.proftext -o %t.profdata && \
+; RUN: opt < %s -verify-ipgo -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+;
+; Ensure copied-metadata diagnostics are suppressed while running inside the
+; PGOInstrumentationUse pass itself (IsPGOUsePass=true).
+
+; Two functions intentionally share the same !prof metadata node.
+define i32 @f1() !prof !10 {
+entry:
+ ret i32 1
+}
+
+define i32 @f2() !prof !10 {
+entry:
+ ret i32 2
+}
+
+; CHECK-LABEL: *** IPGO Verification After PGOInstrumentationUse ***
+; CHECK-NOT: PGOVerify# Copied metadata detected in function
+
+; VERIFY-LABEL: *** IPGO Verification After PGOInstrumentationUse ***
+; VERIFY-NOT: PGOVerify# Copied metadata detected in function
+
+!10 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-zero-func-count.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-zero-func-count.ll
new file mode 100644
index 0000000000000..ab1ab37a17fa2
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-zero-func-count.ll
@@ -0,0 +1,165 @@
+; RUN: opt < %s -verify-ipgo -debug-only=verify-ipgo -passes=loop-unroll -S -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-ipgo -passes=loop-unroll -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
+; REQUIRES: asserts
+
+; CHECK-LABEL: *** IPGO Verification After LoopUnrollPass ***
+; CHECK: PGOVerify# Not able to determine Block frequency for worker, block entry
+
+; VERIFY-LABEL: *** IPGO Verification After LoopUnrollPass ***
+; ModuleID = 'short.ll'
+source_filename = "psimplex.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.network = type { [200 x i8], [200 x i8], i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, double, i64, ptr, ptr, ptr, ptr, ptr, ptr, ptr, i64, i64, i64, i64, i64 }
+%struct.basket = type { ptr, i64, i64, i64 }
+
+ at perm_p = external hidden unnamed_addr global ptr, align 8
+ at basket_sizes = external hidden unnamed_addr global ptr, align 8
+ at basket = external hidden unnamed_addr global ptr, align 8
+ at opt = external hidden unnamed_addr global i1, align 8
+ at opt_basket = external hidden unnamed_addr global ptr, align 8
+
+; Function Attrs: nounwind uwtable
+declare dso_local void @markBaskets(i64 noundef) local_unnamed_addr #0
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none)) #1
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none)) #1
+
+; Function Attrs: nounwind uwtable
+define dso_local void @worker(ptr noundef %net, i32 noundef %thread, i32 noundef %num_threads) local_unnamed_addr #0 {
+entry:
+ %perm = alloca [4061 x ptr], align 16
+ %end_arc = alloca ptr, align 8
+ %arcs1 = getelementptr inbounds nuw %struct.network, ptr %net, i64 0, i32 23
+ %0 = load ptr, ptr %arcs1, align 8, !tbaa !5
+ %stop_arcs2 = getelementptr inbounds nuw %struct.network, ptr %net, i64 0, i32 24
+ %1 = load ptr, ptr %stop_arcs2, align 8, !tbaa !14
+ %m3 = getelementptr inbounds nuw %struct.network, ptr %net, i64 0, i32 5
+ %2 = load i64, ptr %m3, align 8, !tbaa !15
+ %iterations4 = getelementptr inbounds nuw %struct.network, ptr %net, i64 0, i32 28
+ call void @llvm.lifetime.start.p0(i64 32488, ptr nonnull %perm) #3
+ call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %end_arc) #3
+ store ptr %0, ptr %end_arc, align 8, !tbaa !16
+ %3 = load ptr, ptr @basket_sizes, align 8, !tbaa !17
+ %idxprom = sext i32 %thread to i64
+ %arrayidx = getelementptr inbounds i64, ptr %3, i64 %idxprom
+ store i64 0, ptr %arrayidx, align 8, !tbaa !19
+ %div = sdiv i32 4000, %num_threads
+ %add6 = add nsw i32 %div, 61
+ %add7 = add nsw i32 %div, 261
+ %mul = mul nsw i32 %thread, %add7
+ %add8 = add nsw i32 %mul, 1
+ %conv = sext i32 %add8 to i64
+ br label %for.cond
+
+for.cond: ; preds = %for.body, %entry
+ %i.0 = phi i64 [ 1, %entry ], [ %inc, %for.body ]
+ %j.0 = phi i64 [ %conv, %entry ], [ %inc16, %for.body ]
+ %conv12 = sext i32 %add6 to i64
+ %cmp = icmp slt i64 %i.0, %conv12
+ br i1 %cmp, label %for.body, label %while.cond
+
+for.body: ; preds = %for.cond
+ %4 = load ptr, ptr @basket, align 8, !tbaa !20
+ %arrayidx14 = getelementptr inbounds %struct.basket, ptr %4, i64 %j.0
+ %arrayidx15 = getelementptr inbounds nuw [4061 x ptr], ptr %perm, i64 0, i64 %i.0
+ store ptr %arrayidx14, ptr %arrayidx15, align 8, !tbaa !20
+ %inc = add nuw nsw i64 %i.0, 1
+ %inc16 = add nsw i64 %j.0, 1
+ br label %for.cond, !llvm.loop !22
+
+while.cond: ; preds = %if.end, %for.cond
+ %.b = load i1, ptr @opt, align 1
+ br i1 %.b, label %while.end, label %while.body
+
+while.body: ; preds = %while.cond
+ %5 = load ptr, ptr @basket_sizes, align 8, !tbaa !17
+ %6 = load i64, ptr %iterations4, align 8, !tbaa !19
+ %add18 = add nsw i64 %6, %idxprom
+ %conv19 = sext i32 %num_threads to i64
+ %rem = srem i64 %add18, %conv19
+ %max_elems = getelementptr inbounds nuw %struct.network, ptr %net, i64 0, i32 32
+ %7 = load i64, ptr %max_elems, align 8, !tbaa !24
+ %call = call ptr @primal_bea_mpp(i64 noundef %2, ptr noundef %0, ptr noundef %1, ptr noundef %5, ptr noundef nonnull %perm, i32 noundef %thread, ptr noundef nonnull %end_arc, i64 noundef %rem, i64 noundef %conv19, i64 noundef %7) #3
+ %8 = load ptr, ptr @opt_basket, align 8, !tbaa !25
+ %arrayidx22 = getelementptr inbounds ptr, ptr %8, i64 %idxprom
+ store ptr %call, ptr %arrayidx22, align 8, !tbaa !20
+ %add.ptr = getelementptr inbounds nuw ptr, ptr %perm, i64 1
+ %9 = load ptr, ptr @perm_p, align 8, !tbaa !28
+ %arrayidx25 = getelementptr inbounds ptr, ptr %9, i64 %idxprom
+ store ptr %add.ptr, ptr %arrayidx25, align 8, !tbaa !25
+ %cmp26 = icmp eq i32 %thread, 1
+ br i1 %cmp26, label %if.then, label %if.end
+
+if.then: ; preds = %while.body
+ call void @markBaskets(i64 noundef %conv19)
+ br label %if.end
+
+if.end: ; preds = %if.then, %while.body
+ br label %while.cond, !llvm.loop !31
+
+while.end: ; preds = %while.cond
+ call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %end_arc) #3
+ call void @llvm.lifetime.end.p0(i64 32488, ptr nonnull %perm) #3
+ ret void
+}
+
+declare dso_local ptr @primal_bea_mpp(i64 noundef, ptr noundef, ptr noundef, ptr noundef, ptr noundef, i32 noundef, ptr noundef, i64 noundef, i64 noundef, i64 noundef) local_unnamed_addr #2
+
+attributes #0 = { nounwind uwtable "approx-func-fp-math"="true" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="znver4" "target-features"="+adx,+aes,+avx,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+bmi,+bmi2,+clflushopt,+clwb,+clzero,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fsgsbase,+fxsr,+gfni,+invpcid,+lzcnt,+mmx,+movbe,+mwaitx,+pclmul,+pku,+popcnt,+prfchw,+rdpid,+rdpru,+rdrnd,+rdseed,+sahf,+sha,+shstk,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+sse4a,+ssse3,+vaes,+vpclmulqdq,+wbnoinvd,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="true" }
+attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+attributes #2 = { "approx-func-fp-math"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="znver4" "target-features"="+adx,+aes,+avx,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+bmi,+bmi2,+clflushopt,+clwb,+clzero,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fsgsbase,+fxsr,+gfni,+invpcid,+lzcnt,+mmx,+movbe,+mwaitx,+pclmul,+pku,+popcnt,+prfchw,+rdpid,+rdpru,+rdrnd,+rdseed,+sahf,+sha,+shstk,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+sse4a,+ssse3,+vaes,+vpclmulqdq,+wbnoinvd,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="true" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !32}
+!llvm.ident = !{!4}
+
+!32 = !{i32 1, !"ProfileSummary", !33}
+!33 = !{!34, !35, !36, !37, !38, !39, !40, !41}
+!34 = !{!"ProfileFormat", !"InstrProf"}
+!35 = !{!"TotalCount", i64 1}
+!36 = !{!"MaxCount", i64 1}
+!37 = !{!"MaxInternalCount", i64 1}
+!38 = !{!"MaxFunctionCount", i64 1}
+!39 = !{!"NumCounts", i64 1}
+!40 = !{!"NumFunctions", i64 1}
+!41 = !{!"DetailedSummary", !42}
+!42 = !{!43}
+!43 = !{i32 10000, i64 1, i32 1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"uwtable", i32 2}
+!2 = !{i32 1, !"ThinLTO", i32 0}
+!3 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!4 = !{!"clang version 21.1.8 (CLANG: Unknown-Revision)"}
+!5 = !{!6, !13, i64 568, i64 8}
+!6 = !{!7, i64 648, !"network", !7, i64 0, i64 200, !7, i64 200, i64 200, !9, i64 400, i64 8, !9, i64 408, i64 8, !9, i64 416, i64 8, !9, i64 424, i64 8, !9, i64 432, i64 8, !9, i64 440, i64 8, !9, i64 448, i64 8, !9, i64 456, i64 8, !9, i64 464, i64 8, !9, i64 472, i64 8, !9, i64 480, i64 8, !9, i64 488, i64 8, !9, i64 496, i64 8, !9, i64 504, i64 8, !9, i64 512, i64 8, !9, i64 520, i64 8, !9, i64 528, i64 8, !10, i64 536, i64 8, !9, i64 544, i64 8, !11, i64 552, i64 8, !11, i64 560, i64 8, !13, i64 568, i64 8, !13, i64 576, i64 8, !13, i64 584, i64 8, !13, i64 592, i64 8, !13, i64 600, i64 8, !9, i64 608, i64 8, !9, i64 616, i64 8, !9, i64 624, i64 8, !9, i64 632, i64 8, !9, i64 640, i64 8}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C/C++ TBAA"}
+!9 = !{!7, i64 8, !"long"}
+!10 = !{!7, i64 8, !"double"}
+!11 = !{!12, i64 8, !"p1 _ZTS4node"}
+!12 = !{!7, i64 8, !"any pointer"}
+!13 = !{!12, i64 8, !"p1 _ZTS3arc"}
+!14 = !{!6, !13, i64 576, i64 8}
+!15 = !{!6, !9, i64 424, i64 8}
+!16 = !{!13, !13, i64 0, i64 8}
+!17 = !{!18, !18, i64 0, i64 8}
+!18 = !{!12, i64 8, !"p1 long"}
+!19 = !{!9, !9, i64 0, i64 8}
+!20 = !{!21, !21, i64 0, i64 8}
+!21 = !{!12, i64 8, !"p1 _ZTS6basket"}
+!22 = distinct !{!22, !23}
+!23 = !{!"llvm.loop.mustprogress"}
+!24 = !{!6, !9, i64 640, i64 8}
+!25 = !{!26, !26, i64 0, i64 8}
+!26 = !{!27, i64 8, !"p2 _ZTS6basket"}
+!27 = !{!12, i64 8, !"any p2 pointer"}
+!28 = !{!29, !29, i64 0, i64 8}
+!29 = !{!30, i64 8, !"p3 _ZTS6basket"}
+!30 = !{!27, i64 8, !"any p3 pointer"}
+!31 = distinct !{!31, !23}
>From 05c1c76d1f4089cbaceea0da1c2819bd2a810fdf Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Tue, 5 May 2026 11:42:51 +0530
Subject: [PATCH 4/5] [PGOVerify] [4/5] Add -verify-ipgo-funcs-filter option
Implements function filtering to skip PGO verification for specified functions
(e.g., externally-available code, template instantiations). Provides fine-grained
control over validation scope.
---
llvm/include/llvm/Transforms/IPO/PGOVerify.h | 5 ++
llvm/lib/Transforms/IPO/PGOVerify.cpp | 28 ++++++++-
.../verify-ipgo-funcs-filter-basic.ll | 59 +++++++++++++++++++
...erify-ipgo-funcs-filter-externally-skip.ll | 26 ++++++++
.../verify-ipgo-funcs-filter-list.ll | 59 +++++++++++++++++++
5 files changed, 176 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-externally-skip.ll
create mode 100644 llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOVerify.h b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
index 945aa354b6469..d185eb331539e 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
@@ -74,6 +74,11 @@ class IPGOVerifier {
/// Invalidate cached block-frequency entries for changed IR scopes.
void invalidateFunctionFrequencyCache(Any IR);
+ /// Return true if a function is eligible for verification.
+ ///
+ /// Applies verifier-local exclusions and optional command-line filtering.
+ bool shouldVerifyFunction(const Function *F) const;
+
/// Handle module callbacks by delegating each function to function handler.
void runAfterPass(const Module *M);
diff --git a/llvm/lib/Transforms/IPO/PGOVerify.cpp b/llvm/lib/Transforms/IPO/PGOVerify.cpp
index 3307a1776fe68..3e96cd1f9675f 100644
--- a/llvm/lib/Transforms/IPO/PGOVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOVerify.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/IPO/PGOVerify.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -38,6 +39,10 @@ static cl::opt<bool>
VerifyIPGO("verify-ipgo", cl::init(false), cl::Hidden,
cl::desc("Enable Instrumented PGO verification"));
+static cl::list<std::string>
+ VerifyIPGOFuncList("verify-ipgo-funcs", cl::Hidden,
+ cl::desc("Comma-separated list of functions to verify"));
+
/// Emit a labelled PGO-verify diagnostic to stderr (when enabled).
static void emitPGOVerifyDiagnostic(const Function *F, StringRef Kind,
const std::string &Msg) {
@@ -46,6 +51,24 @@ static void emitPGOVerifyDiagnostic(const Function *F, StringRef Kind,
<< ": " << Msg << "\n";
}
+bool IPGOVerifier::shouldVerifyFunction(const Function *F) const {
+ if (!F || F->isDeclaration())
+ return false;
+
+ if (F->hasAvailableExternallyLinkage())
+ return false;
+
+ // Cache command-line function filters.
+ static const DenseSet<StringRef> FuncFilter = [] {
+ DenseSet<StringRef> S;
+ for (const auto &Func : VerifyIPGOFuncList)
+ S.insert(Func);
+ return S;
+ }();
+
+ return FuncFilter.empty() || FuncFilter.count(F->getName());
+}
+
/// Register post-pass diagnostic callbacks for `-verify-ipgo`.
///
/// \param PIC Pass instrumentation callback registry.
@@ -159,7 +182,7 @@ void IPGOVerifier::runAfterPass(const Module *M) {
// Then run validations using the populated cache.
for (const Function &F : *M) {
- if (F.isDeclaration())
+ if (!shouldVerifyFunction(&F))
continue;
validateBlockFrequencies(&F);
validateEntryCountAgainstCallerSum(&F);
@@ -173,6 +196,9 @@ void IPGOVerifier::runAfterPass(Function *F) {
if (!F || F->isDeclaration() || !F->getParent())
return;
+ if (!shouldVerifyFunction(F))
+ return;
+
// Run Use-phase checks only when an InstrProf use summary is present.
if (F->getParent()->getProfileSummary(/*IsCS=*/true))
return;
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll
new file mode 100644
index 0000000000000..f504e5e21839e
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -verify-ipgo -verify-ipgo-print-diagnostics -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=NOFILTER
+; RUN: opt < %s -verify-ipgo -verify-ipgo-print-diagnostics -verify-ipgo-funcs=callee_a -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=FILTERA
+; RUN: opt < %s -verify-ipgo -verify-ipgo-print-diagnostics -verify-ipgo-funcs=main -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=FILTERMAIN
+; REQUIRES: asserts
+;
+; Verify that -verify-ipgo-funcs restricts verification to named functions.
+; - Without a filter, both callee_a and callee_b mismatches are reported.
+; - With -verify-ipgo-funcs=callee_a, only callee_a is verified/reported.
+; - With -verify-ipgo-funcs=main, no callee mismatch is reported.
+
+define internal i32 @callee_a(i32 %x) !prof !10 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define internal i32 @callee_b(i32 %x) !prof !11 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define i32 @main() !prof !12 {
+entry:
+ %a = call i32 @callee_a(i32 7)
+ %b = call i32 @callee_b(i32 9)
+ %s = add i32 %a, %b
+ ret i32 %s
+}
+
+; NOFILTER: PGOVerify# EntryCountMismatch in function callee_a: Entry count mismatch: entry=2 vs caller-sum=1
+; NOFILTER: PGOVerify# EntryCountMismatch in function callee_b: Entry count mismatch: entry=3 vs caller-sum=1
+
+; FILTERA: PGOVerify# EntryCountMismatch in function callee_a: Entry count mismatch: entry=2 vs caller-sum=1
+; FILTERA-NOT: PGOVerify# EntryCountMismatch in function callee_b: Entry count mismatch: entry=3 vs caller-sum=1
+
+; FILTERMAIN: *** IPGO Verification After
+; FILTERMAIN-NOT: PGOVerify# EntryCountMismatch
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 6}
+!34 = !{!"MaxCount", i64 3}
+!35 = !{!"MaxInternalCount", i64 3}
+!36 = !{!"MaxFunctionCount", i64 3}
+!37 = !{!"NumCounts", i64 3}
+!38 = !{!"NumFunctions", i64 3}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 3, i32 1}
+!42 = !{i32 999000, i64 2, i32 2}
+!43 = !{i32 999999, i64 1, i32 3}
+
+; Entry counts intentionally mismatch caller-site sum (=1 each call site in main)
+!10 = !{!"function_entry_count", i64 2}
+!11 = !{!"function_entry_count", i64 3}
+!12 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-externally-skip.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-externally-skip.ll
new file mode 100644
index 0000000000000..93cfea8429617
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-externally-skip.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -verify-ipgo -verify-ipgo-print-diagnostics -verify-ipgo-funcs=ext_callee -passes=instcombine -S -disable-output 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+;
+; Verify that -verify-ipgo-funcs still honors verifier exclusions.
+; Even when explicitly listed, available_externally functions are skipped.
+;
+; CHECK: *** IPGO Verification After
+; CHECK-NOT: PGOVerify# Entry count mismatch in function ext_callee
+
+define available_externally i32 @ext_callee(i32 %x) !prof !10 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define i32 @main() !prof !11 {
+entry:
+ %r = call i32 @ext_callee(i32 7), !prof !12
+ ret i32 %r
+}
+
+; Intentionally mismatched if ext_callee were verified: entry=2, caller-sum=1.
+!10 = !{!"function_entry_count", i64 2}
+!11 = !{!"function_entry_count", i64 1}
+!12 = !{!"VP", i32 0, i64 1, i64 123456789, i64 1}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll
new file mode 100644
index 0000000000000..a62ac6e9392c8
--- /dev/null
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -verify-ipgo -verify-ipgo-print-diagnostics -verify-ipgo-funcs=callee_a -verify-ipgo-funcs=callee_b -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=LISTAB
+; RUN: opt < %s -verify-ipgo -verify-ipgo-print-diagnostics -verify-ipgo-funcs=callee_a -verify-ipgo-funcs=missing_fn -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=LISTA
+; RUN: opt < %s -verify-ipgo -verify-ipgo-print-diagnostics -verify-ipgo-funcs=missing_fn -passes=instcombine -S -disable-output 2>&1 | FileCheck %s --check-prefix=NOMATCH
+; REQUIRES: asserts
+;
+; Verify list behavior of -verify-ipgo-funcs.
+; - Repeating -verify-ipgo-funcs adds multiple names to the filter.
+; - Unknown names are ignored.
+; - A list with no matching function names emits no entry-count mismatch.
+
+define internal i32 @callee_a(i32 %x) !prof !10 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define internal i32 @callee_b(i32 %x) !prof !11 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define i32 @main() !prof !12 {
+entry:
+ %a = call i32 @callee_a(i32 7)
+ %b = call i32 @callee_b(i32 9)
+ %s = add i32 %a, %b
+ ret i32 %s
+}
+
+; LISTAB: PGOVerify# EntryCountMismatch in function callee_a: Entry count mismatch: entry=2 vs caller-sum=1
+; LISTAB: PGOVerify# EntryCountMismatch in function callee_b: Entry count mismatch: entry=3 vs caller-sum=1
+
+; LISTA: PGOVerify# EntryCountMismatch in function callee_a: Entry count mismatch: entry=2 vs caller-sum=1
+; LISTA-NOT: PGOVerify# EntryCountMismatch in function callee_b: Entry count mismatch: entry=3 vs caller-sum=1
+
+; NOMATCH: *** IPGO Verification After
+; NOMATCH-NOT: PGOVerify# EntryCountMismatch
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 6}
+!34 = !{!"MaxCount", i64 3}
+!35 = !{!"MaxInternalCount", i64 3}
+!36 = !{!"MaxFunctionCount", i64 3}
+!37 = !{!"NumCounts", i64 3}
+!38 = !{!"NumFunctions", i64 3}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 3, i32 1}
+!42 = !{i32 999000, i64 2, i32 2}
+!43 = !{i32 999999, i64 1, i32 3}
+
+; Entry counts intentionally mismatch caller-site sum (=1 each call site in main)
+!10 = !{!"function_entry_count", i64 2}
+!11 = !{!"function_entry_count", i64 3}
+!12 = !{!"function_entry_count", i64 1}
>From 24aaa56843661f6631f216ff04e20134457fe98e Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Tue, 5 May 2026 11:31:16 +0530
Subject: [PATCH 5/5] [PGOVerify] [5/5] Add validation for PGOGen
instrumentation pass
Extends IPGOVerifier to validate counter initialization and metadata correctness
during pgo-instr-gen pass. Complements existing pgo-instr-use validation.
---
llvm/include/llvm/Transforms/IPO/PGOVerify.h | 7 ++
llvm/lib/Transforms/IPO/PGOVerify.cpp | 91 +++++++++++++++++--
.../verify-ipgo-funcs-filter-basic.ll | 10 +-
.../verify-ipgo-funcs-filter-list.ll | 10 +-
.../verify-ipgo-gen-counter-load-mismatch.ll | 5 +-
.../PGOVerifier/verify-ipgo-merge-function.ll | 2 +-
6 files changed, 104 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/Transforms/IPO/PGOVerify.h b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
index d185eb331539e..0e5bdf1c148ad 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOVerify.h
@@ -122,6 +122,13 @@ class IPGOVerifier {
/// callsites to the function have extractable profile totals.
void validateEntryCountAgainstCallerSum(const Function *F);
+ /// Validate instrumentation-generation phase invariants.
+ ///
+ /// Checks for gen-phase violations such as:
+ /// - InstrProf intrinsic names matching their containing function
+ /// - Counter global loads from the correct function
+ void verifyGenPhase(const Function *F);
+
/// Per-instance cache of inferred block-frequency data keyed by function.
DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
};
diff --git a/llvm/lib/Transforms/IPO/PGOVerify.cpp b/llvm/lib/Transforms/IPO/PGOVerify.cpp
index 3e96cd1f9675f..6dedf0093cf21 100644
--- a/llvm/lib/Transforms/IPO/PGOVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOVerify.cpp
@@ -18,9 +18,11 @@
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/ProfileSummary.h"
+#include "llvm/ProfileData/InstrProf.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <limits>
@@ -43,12 +45,19 @@ static cl::list<std::string>
VerifyIPGOFuncList("verify-ipgo-funcs", cl::Hidden,
cl::desc("Comma-separated list of functions to verify"));
-/// Emit a labelled PGO-verify diagnostic to stderr (when enabled).
-static void emitPGOVerifyDiagnostic(const Function *F, StringRef Kind,
- const std::string &Msg) {
+/// Emit PGO verification diagnostics with structured formatting.
+///
+/// \param F Function being verified.
+/// \param RemarkName Diagnostic remark identifier.
+/// \param Msg Error/diagnostic message.
+static void emitPGOVerifyDiagnostic(const Function *F, StringRef RemarkName,
+ const Twine &Msg) {
+ std::string MsgText = Msg.str();
if (VerifyIPGOPrintDiagnostics)
- errs() << "PGOVerify# " << Kind << " in function " << F->getName()
- << ": " << Msg << "\n";
+ errs() << "PGOVerify[" << RemarkName << "] " << F->getName() << ": "
+ << MsgText << "\n";
+ LLVM_DEBUG(dbgs() << "PGOVerify[" << RemarkName << "] " << F->getName()
+ << ": " << MsgText << "\n");
}
bool IPGOVerifier::shouldVerifyFunction(const Function *F) const {
@@ -161,9 +170,14 @@ void IPGOVerifier::runAfterPass(const Module *M) {
// Run Use-phase checks only when an InstrProf use summary is present.
if (M->getProfileSummary(/*IsCS=*/true))
return;
- if (!hasInstrProfUseSummary(M))
+ if (!hasInstrProfUseSummary(M)) {
+ for (const Function &F : *M) {
+ if (F.isDeclaration())
+ continue;
+ verifyGenPhase(&F);
+ }
return;
-
+ }
// First build frequency cache for all non-declaration functions so caller
// information is available regardless of function order in the module.
for (const Function &F : *M) {
@@ -202,8 +216,11 @@ void IPGOVerifier::runAfterPass(Function *F) {
// Run Use-phase checks only when an InstrProf use summary is present.
if (F->getParent()->getProfileSummary(/*IsCS=*/true))
return;
- if (!hasInstrProfUseSummary(F->getParent()))
+ if (!hasInstrProfUseSummary(F->getParent())) {
+ // Run Gen-phase checks (no dependencies on other passes).
+ verifyGenPhase(F);
return;
+ }
// Rebuild the minimal local analysis stack here so verification can query
// non-synthetic block profile counts after each pass callback.
@@ -425,6 +442,64 @@ bool IPGOVerifier::hasFunctionLocalCountOverflow(const Function *F,
return false;
}
+/// Validate instrumentation-generation phase invariants.
+void IPGOVerifier::verifyGenPhase(const Function *F) {
+ // Validate instrprof_increment names against the containing function.
+ auto *IncIntrinsic = Intrinsic::getOrInsertDeclaration(
+ const_cast<Module *>(F->getParent()), Intrinsic::instrprof_increment);
+
+ if (IncIntrinsic) {
+ for (User *U : IncIntrinsic->users()) {
+ auto *Instr = dyn_cast<InstrProfCntrInstBase>(U);
+ if (!Instr || Instr->getFunction() != F)
+ continue;
+
+ StringRef Prefix = getInstrProfNameVarPrefix();
+ StringRef ProfiledName =
+ cast<InstrProfInstBase>(Instr)->getName()->getName().substr(
+ Prefix.size());
+
+ if (!ProfiledName.ends_with(F->getName())) {
+ LLVM_DEBUG(dbgs() << "PGOVerify# Intrinsic name mismatch in function "
+ << F->getName() << ": ");
+ LLVM_DEBUG(Instr->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
+ emitPGOVerifyDiagnostic(F, "IntrinsicNameMismatch",
+ "Intrinsic name mismatch: profiling " +
+ ProfiledName.str() + " instead of " +
+ F->getName().str());
+ }
+ }
+ }
+
+ // Validate counter-global loads against the containing function.
+ for (auto &GV : const_cast<Module *>(F->getParent())->globals()) {
+
+ StringRef Prefix = getInstrProfCountersVarPrefix();
+ if (!GV.getName().starts_with(Prefix))
+ continue;
+
+ StringRef CounterName = GV.getName().substr(Prefix.size());
+
+ for (const User *U : GV.users()) {
+ auto *LI = dyn_cast<LoadInst>(U);
+ if (!LI || LI->getFunction() != F)
+ continue;
+
+ if (!CounterName.contains(F->getName())) {
+ LLVM_DEBUG(dbgs() << "PGOVerify# Counter load mismatch in function "
+ << F->getName() << ": ");
+ LLVM_DEBUG(LI->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
+ emitPGOVerifyDiagnostic(F, "CounterLoadMismatch",
+ "Counter variable mismatch: loading " +
+ CounterName.str() + " instead of " +
+ F->getName().str());
+ }
+ }
+ }
+}
+
bool IPGOVerifier::hasInstrProfUseSummary(const Module *M) const {
if (!M)
return false;
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll
index f504e5e21839e..db9758a183578 100644
--- a/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-basic.ll
@@ -28,14 +28,14 @@ entry:
ret i32 %s
}
-; NOFILTER: PGOVerify# EntryCountMismatch in function callee_a: Entry count mismatch: entry=2 vs caller-sum=1
-; NOFILTER: PGOVerify# EntryCountMismatch in function callee_b: Entry count mismatch: entry=3 vs caller-sum=1
+; NOFILTER: PGOVerify[EntryCountMismatch] callee_a: Entry count mismatch: entry=2 vs caller-sum=1
+; NOFILTER: PGOVerify[EntryCountMismatch] callee_b: Entry count mismatch: entry=3 vs caller-sum=1
-; FILTERA: PGOVerify# EntryCountMismatch in function callee_a: Entry count mismatch: entry=2 vs caller-sum=1
-; FILTERA-NOT: PGOVerify# EntryCountMismatch in function callee_b: Entry count mismatch: entry=3 vs caller-sum=1
+; FILTERA: PGOVerify[EntryCountMismatch] callee_a: Entry count mismatch: entry=2 vs caller-sum=1
+; FILTERA-NOT: PGOVerify[EntryCountMismatch] callee_b: Entry count mismatch: entry=3 vs caller-sum=1
; FILTERMAIN: *** IPGO Verification After
-; FILTERMAIN-NOT: PGOVerify# EntryCountMismatch
+; FILTERMAIN-NOT: PGOVerify[EntryCountMismatch]
!llvm.module.flags = !{!30}
!30 = !{i32 1, !"ProfileSummary", !31}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll
index a62ac6e9392c8..57e9e847fd550 100644
--- a/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-funcs-filter-list.ll
@@ -28,14 +28,14 @@ entry:
ret i32 %s
}
-; LISTAB: PGOVerify# EntryCountMismatch in function callee_a: Entry count mismatch: entry=2 vs caller-sum=1
-; LISTAB: PGOVerify# EntryCountMismatch in function callee_b: Entry count mismatch: entry=3 vs caller-sum=1
+; LISTAB: PGOVerify[EntryCountMismatch] callee_a: Entry count mismatch: entry=2 vs caller-sum=1
+; LISTAB: PGOVerify[EntryCountMismatch] callee_b: Entry count mismatch: entry=3 vs caller-sum=1
-; LISTA: PGOVerify# EntryCountMismatch in function callee_a: Entry count mismatch: entry=2 vs caller-sum=1
-; LISTA-NOT: PGOVerify# EntryCountMismatch in function callee_b: Entry count mismatch: entry=3 vs caller-sum=1
+; LISTA: PGOVerify[EntryCountMismatch] callee_a: Entry count mismatch: entry=2 vs caller-sum=1
+; LISTA-NOT: PGOVerify[EntryCountMismatch] callee_b: Entry count mismatch: entry=3 vs caller-sum=1
; NOMATCH: *** IPGO Verification After
-; NOMATCH-NOT: PGOVerify# EntryCountMismatch
+; NOMATCH-NOT: PGOVerify[EntryCountMismatch]
!llvm.module.flags = !{!30}
!30 = !{i32 1, !"ProfileSummary", !31}
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll
index d790da8c17fde..cf67754719d7e 100644
--- a/llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-gen-counter-load-mismatch.ll
@@ -2,8 +2,8 @@
; RUN: opt < %s -verify-ipgo -passes=pgo-instr-gen -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
; REQUIRES: asserts
;
-; Ensure verify-ipgo runs in Gen phase without emitting entry/block diagnostics
-; for this minimal IR.
+; Ensure Gen-phase counter load mismatch is diagnosed when a function loads
+; a counter global that belongs to a different function.
@__profc_bar = global i64 0, align 8
@@ -21,3 +21,4 @@ entry:
; VERIFY-LABEL: *** IPGO Verification After PGOInstrumentationGen ***
; VERIFY-NOT: PGOVerify# Entry count mismatch
; VERIFY-NOT: PGOVerify# Block frequency mismatch
+; VERIFY: PGOVerify[CounterLoadMismatch] foo: Counter variable mismatch: loading bar instead of foo
diff --git a/llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll b/llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll
index 1a29d919c4291..25ebd924352cb 100644
--- a/llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll
+++ b/llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll
@@ -1,7 +1,7 @@
; RUN: opt < %s -verify-ipgo -passes=mergefunc -S -disable-output 2>&1 | FileCheck %s --check-prefix=VERIFY
; VERIFY: *** IPGO Verification After MergeFunctionsPass ***
-; VERIFY-NEXT: PGOVerify# EntryCountMismatch in function add1: Entry count mismatch: entry=1 vs caller-sum=2
+; VERIFY-NEXT: PGOVerify[EntryCountMismatch] add1: Entry count mismatch: entry=1 vs caller-sum=2
; ModuleID = '../llvm/test/Transforms/PGOVerifier/verify-ipgo-merge-function.ll'
source_filename = "mymergefun.c"
More information about the llvm-commits
mailing list