[clang] [llvm] [PGOVerify] Add end-to-end IPGOVerifier infrastructure, validations, and function filtering (PR #196295)
Alok Kumar Sharma via cfe-commits
cfe-commits at lists.llvm.org
Thu Sep 10 02:47:54 PDT 2026
https://github.com/alokkrsharma updated https://github.com/llvm/llvm-project/pull/196295
>From bf11055e99809abfc0810c6df9727580b6e49a75 Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:02:37 +0530
Subject: [PATCH 01/13] [PGOFlowVerify] Add hook and standalone verify-pgo-flow
pass
Register PGOFlowVerifier with StandardInstrumentations
(-verify-pgo-flow) and as a pipeline pass. The hook is opt-in.
-passes=verify-pgo-flow walks the module; function(verify-pgo-flow)
walks one function. Unchanged passes are skipped. No InstrProf
use-phase summary means the walk does nothing. Banners go to stderr
(-verify-pgo-flow-print-diagnostics=false silences them).
-debug-only=verify-pgo-flow logs hook plumbing.
---
.../llvm/Passes/StandardInstrumentations.h | 4 +
.../llvm/Transforms/IPO/PGOFlowVerify.h | 58 ++++++
llvm/lib/Passes/PassBuilder.cpp | 1 +
llvm/lib/Passes/PassRegistry.def | 2 +
llvm/lib/Passes/StandardInstrumentations.cpp | 7 +
llvm/lib/Transforms/IPO/CMakeLists.txt | 1 +
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 169 ++++++++++++++++++
.../PGOFlowVerifier/verify-pgo-flow-fatal.ll | 11 ++
.../PGOFlowVerifier/verify-pgo-flow-hook.ll | 47 +++++
.../verify-pgo-flow-no-summary.ll | 13 ++
.../verify-pgo-flow-sample-profile.ll | 27 +++
.../verify-pgo-flow-standalone-pass.ll | 25 +++
12 files changed, 365 insertions(+)
create mode 100644 llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
create mode 100644 llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-fatal.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-hook.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-no-summary.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-sample-profile.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-standalone-pass.ll
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index beaad14201334..e4c904d7c01a7 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -32,6 +32,7 @@
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Transforms/IPO/SampleProfileProbe.h"
+#include <memory>
#include <string>
#include <utility>
@@ -41,6 +42,7 @@ class Module;
class Function;
class MachineFunction;
class PassInstrumentationCallbacks;
+class PGOFlowVerifier;
/// Instrumentation to print IR before/after passes.
///
@@ -619,6 +621,7 @@ class StandardInstrumentations {
IRChangedTester ChangeTester;
VerifyInstrumentation Verify;
DroppedVariableStatsIR DroppedStatsIR;
+ std::unique_ptr<PGOFlowVerifier> PGOFlowVerification;
bool VerifyEach;
@@ -627,6 +630,7 @@ class StandardInstrumentations {
StandardInstrumentations(LLVMContext &Context, bool DebugLogging,
bool VerifyEach = false,
PrintPassOptions PrintPassOpts = PrintPassOptions());
+ LLVM_ABI ~StandardInstrumentations();
// Register all the standard instrumentation callbacks. If \p FAM is nullptr
// then PreservedCFGChecker is not enabled.
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
new file mode 100644
index 0000000000000..9704f5e3b2b98
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -0,0 +1,58 @@
+//===- PGOFlowVerify.h - PGO flow verification ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// InstrProf flow checks. `-verify-pgo-flow` runs after transforming passes;
+/// the `verify-pgo-flow` pass is the same walk on demand. Adaptors and pass
+/// managers are skipped so a loop nest is not re-walked per adaptor.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_PGOFLOWVERIFY_H
+#define LLVM_TRANSFORMS_IPO_PGOFLOWVERIFY_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/LazyCallGraph.h"
+#include "llvm/IR/IRUnitRef.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+class Function;
+class Loop;
+class Module;
+class PassInstrumentationCallbacks;
+
+/// Walk IR after transforms so InstrProf use-phase flow checks can run.
+class PGOFlowVerifier {
+public:
+ /// True when `-verify-pgo-flow` is set.
+ LLVM_ABI static bool isHookEnabled();
+ LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC);
+ LLVM_ABI void runAfterPass(StringRef PassID, IRUnitRef IR);
+
+private:
+ void runAfterPass(const Module *M);
+ void runAfterPass(const Function *F);
+ void runAfterPass(const LazyCallGraph::SCC *C);
+ void runAfterPass(const Loop *L);
+ bool hasInstrProfUseSummary(const Module *M) const;
+};
+
+/// Pipeline pass that runs the same walk as the `-verify-pgo-flow` hook.
+class PGOFlowVerifierPass : public RequiredPassInfoMixin<PGOFlowVerifierPass> {
+public:
+ LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+ LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+
+private:
+ PGOFlowVerifier Verifier;
+};
+
+} // end namespace llvm
+#endif // LLVM_TRANSFORMS_IPO_PGOFLOWVERIFY_H
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 725ce2d589a31..fc4c54dad9666 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -261,6 +261,7 @@
#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
#include "llvm/Transforms/IPO/MergeFunctions.h"
#include "llvm/Transforms/IPO/OpenMPOpt.h"
+#include "llvm/Transforms/IPO/PGOFlowVerify.h"
#include "llvm/Transforms/IPO/PartialInlining.h"
#include "llvm/Transforms/IPO/SCCP.h"
#include "llvm/Transforms/IPO/SampleProfile.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 43a15f4cf9ff3..216e39d1830c7 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -185,6 +185,7 @@ MODULE_PASS("trigger-verifier-error", TriggerVerifierErrorPass())
MODULE_PASS("tsan-module", ModuleThreadSanitizerPass())
MODULE_PASS("tysan", TypeSanitizerPass())
MODULE_PASS("verify", VerifierPass())
+MODULE_PASS("verify-pgo-flow", PGOFlowVerifierPass())
MODULE_PASS("view-callgraph", CallGraphViewerPass())
MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass())
#undef MODULE_PASS
@@ -562,6 +563,7 @@ FUNCTION_PASS("unify-loop-exits", UnifyLoopExitsPass())
FUNCTION_PASS("unreachableblockelim", UnreachableBlockElimPass())
FUNCTION_PASS("vector-combine", VectorCombinePass())
FUNCTION_PASS("verify", VerifierPass())
+FUNCTION_PASS("verify-pgo-flow", PGOFlowVerifierPass())
FUNCTION_PASS("verify<cycles>", CycleInfoVerifierPass())
FUNCTION_PASS("verify<domtree>", DominatorTreeVerifierPass())
FUNCTION_PASS("verify<loops>", LoopVerifierPass())
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 882df3d0d6abd..65ea487bac004 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -44,6 +44,7 @@
#include "llvm/Support/Regex.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/IPO/PGOFlowVerify.h"
#include <utility>
#include <vector>
@@ -2487,6 +2488,8 @@ StandardInstrumentations::StandardInstrumentations(
Verify(DebugLogging), DroppedStatsIR(DroppedVarStats),
VerifyEach(VerifyEach) {}
+StandardInstrumentations::~StandardInstrumentations() = default;
+
PrintCrashIRInstrumentation *PrintCrashIRInstrumentation::CrashReporter =
nullptr;
@@ -2554,6 +2557,10 @@ void StandardInstrumentations::registerCallbacks(
OptPassGate.registerCallbacks(PIC);
PrintChangedIR.registerCallbacks(PIC);
PseudoProbeVerification.registerCallbacks(PIC);
+ if (PGOFlowVerifier::isHookEnabled()) {
+ PGOFlowVerification = std::make_unique<PGOFlowVerifier>();
+ PGOFlowVerification->registerCallbacks(PIC);
+ }
if (VerifyEach)
Verify.registerCallbacks(PIC, MAM);
PrintChangedDiff.registerCallbacks(PIC);
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index fd791f9fe86f8..2cbb3bda6f25a 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -37,6 +37,7 @@ add_llvm_component_library(LLVMipo
ModuleInliner.cpp
OpenMPOpt.cpp
PartialInlining.cpp
+ PGOFlowVerify.cpp
SampleContextTracker.cpp
SampleProfile.cpp
SampleProfileMatcher.cpp
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
new file mode 100644
index 0000000000000..d24d9aa2d94ed
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -0,0 +1,169 @@
+//===- PGOFlowVerify.cpp - PGO flow verification -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// After-pass hook and standalone pass. Flag behavior is on the cl::opt
+// definitions below.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/PGOFlowVerify.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfileSummary.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+#include <vector>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "verify-pgo-flow"
+
+static cl::opt<bool> VerifyPGOFlow(
+ "verify-pgo-flow", cl::init(false), cl::Hidden,
+ cl::desc("Run InstrProf flow checks after IR-changing passes"));
+
+static cl::opt<bool> VerifyPGOFlowPrintDiagnostics(
+ "verify-pgo-flow-print-diagnostics", cl::init(true), cl::Hidden,
+ cl::desc("Print verify-pgo-flow banners and findings to stderr"));
+
+static cl::opt<bool> VerifyPGOFlowFatal(
+ "verify-pgo-flow-fatal", cl::init(false), cl::Hidden,
+ cl::desc("Abort after a flow-check finding (no-op until checks land)"));
+
+static void printVerifyBanner(StringRef PassName, bool Skipped) {
+ if (!VerifyPGOFlowPrintDiagnostics)
+ return;
+ errs() << "*** PGO Flow Verification After " << PassName
+ << (Skipped ? " (Skipped)" : "") << " ***\n";
+}
+
+bool PGOFlowVerifier::isHookEnabled() { return VerifyPGOFlow; }
+
+void PGOFlowVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
+ if (!VerifyPGOFlow) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: hook not registered "
+ "(-verify-pgo-flow is off)\n");
+ return;
+ }
+
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: registering after-pass hook\n");
+ PIC.registerAfterPassCallback([this](StringRef PassName, IRUnitRef IR,
+ const PreservedAnalyses &PA) {
+ // Same ignore list as print-changed. Adaptors would re-walk the
+ // function after every nested loop pass.
+ static const std::vector<StringRef> Ignored = {"PassManager",
+ "PassAdaptor",
+ "AnalysisManagerProxy",
+ "DevirtSCCRepeatedPass",
+ "ModuleInlinerWrapperPass",
+ "VerifierPass",
+ "PrintModulePass",
+ "PrintMIRPass",
+ "PrintMIRPreparePass",
+ "RequireAnalysisPass",
+ "InvalidateAnalysisPass"};
+ if (isSpecialPass(PassName, Ignored)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: after " << PassName
+ << " (skip, ignored pass)\n");
+ return;
+ }
+ bool Changed = !PA.areAllPreserved();
+ LLVM_DEBUG(
+ dbgs() << "PGOFlowVerifier: after " << PassName
+ << (Changed ? " (walk)\n" : " (skip, PA all preserved)\n"));
+ printVerifyBanner(PassName, /*Skipped=*/!Changed);
+ if (!Changed)
+ return;
+ runAfterPass(PassName, IR);
+ });
+}
+
+void PGOFlowVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
+ if (const auto *M = dyn_cast<Module>(IR)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: module IR after " << PassID << "\n");
+ runAfterPass(M);
+ } else if (const auto *F = dyn_cast<Function>(IR)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: function IR '" << F->getName()
+ << "' after " << PassID << "\n");
+ runAfterPass(F);
+ } else if (const auto *C = dyn_cast<LazyCallGraph::SCC>(IR)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: SCC IR after " << PassID << "\n");
+ runAfterPass(C);
+ } else if (const auto *L = dyn_cast<Loop>(IR)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: loop IR after " << PassID << "\n");
+ runAfterPass(L);
+ } else {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: unhandled IR unit after " << PassID
+ << "\n");
+ }
+}
+
+void PGOFlowVerifier::runAfterPass(const Module *M) {
+ if (!M)
+ return;
+ for (const Function &F : *M)
+ runAfterPass(&F);
+}
+
+void PGOFlowVerifier::runAfterPass(const Function *F) {
+ if (!F || F->isDeclaration())
+ return;
+ if (!hasInstrProfUseSummary(F->getParent())) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip '" << F->getName()
+ << "' (no InstrProf use-phase summary)\n");
+ return;
+ }
+}
+
+bool PGOFlowVerifier::hasInstrProfUseSummary(const Module *M) const {
+ if (!M)
+ return false;
+ Metadata *SummaryMD = M->getProfileSummary(/*IsCS=*/false);
+ if (!SummaryMD)
+ return false;
+ std::unique_ptr<ProfileSummary> PS(ProfileSummary::getFromMD(SummaryMD));
+ return PS && (PS->getKind() == ProfileSummary::PSK_Instr ||
+ PS->getKind() == ProfileSummary::PSK_CSInstr);
+}
+
+void PGOFlowVerifier::runAfterPass(const LazyCallGraph::SCC *C) {
+ if (!C)
+ return;
+ for (const LazyCallGraph::Node &N : *C)
+ runAfterPass(&N.getFunction());
+}
+
+void PGOFlowVerifier::runAfterPass(const Loop *L) {
+ if (!L)
+ return;
+ runAfterPass(L->getHeader()->getParent());
+}
+
+PreservedAnalyses PGOFlowVerifierPass::run(Module &M,
+ ModuleAnalysisManager &MAM) {
+ (void)MAM;
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: pipeline pass (module)\n");
+ printVerifyBanner("verify-pgo-flow", /*Skipped=*/false);
+ Verifier.runAfterPass("verify-pgo-flow", M);
+ return PreservedAnalyses::all();
+}
+
+PreservedAnalyses PGOFlowVerifierPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ (void)FAM;
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: pipeline pass (function "
+ << F.getName() << ")\n");
+ printVerifyBanner("verify-pgo-flow", /*Skipped=*/false);
+ Verifier.runAfterPass("verify-pgo-flow", F);
+ return PreservedAnalyses::all();
+}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-fatal.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-fatal.ll
new file mode 100644
index 0000000000000..cc39c174693d5
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-fatal.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -passes=instcombine -verify-pgo-flow \
+; RUN: -verify-pgo-flow-fatal -disable-output 2>&1 | FileCheck %s
+;
+; No findings yet, so -verify-pgo-flow-fatal must not abort.
+
+; CHECK: *** PGO Flow Verification After InstCombinePass ***{{$}}
+
+define i32 @f(i32 %x) {
+ %a = add i32 %x, 0
+ ret i32 %a
+}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-hook.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-hook.ll
new file mode 100644
index 0000000000000..1b6854e26da11
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-hook.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -passes=instcombine -verify-pgo-flow -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=ON
+; RUN: opt < %s -passes=instcombine -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=OFF --allow-empty
+; RUN: opt < %s -passes='instcombine,instcombine' -verify-pgo-flow \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=TWICE
+; RUN: opt < %s -passes=instcombine -verify-pgo-flow \
+; RUN: -verify-pgo-flow-print-diagnostics=false -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=QUIET --allow-empty
+; RUN: opt < %s -passes='loop(indvars)' -verify-pgo-flow -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=LOOP
+;
+; Off unless -verify-pgo-flow. Unchanged passes are skipped. Loop adaptors
+; are not walked.
+
+; ON: *** PGO Flow Verification After InstCombinePass ***{{$}}
+
+; OFF-NOT: PGO Flow Verification
+
+; TWICE: *** PGO Flow Verification After InstCombinePass ***{{$}}
+; TWICE: *** PGO Flow Verification After InstCombinePass (Skipped) ***
+
+; QUIET-NOT: PGO Flow Verification
+
+; LOOP-NOT: PassAdaptor
+; LOOP: After IndVarSimplifyPass
+
+define i32 @f(i32 %x) {
+ %a = add i32 %x, 0
+ ret i32 %a
+}
+
+define void @loop(ptr %p, i32 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %for.body ]
+ %slot = getelementptr i32, ptr %p, i32 %i
+ store i32 %i, ptr %slot
+ %i.next = add nsw i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %for.body, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-no-summary.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-no-summary.ll
new file mode 100644
index 0000000000000..d91d992e8a39e
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-no-summary.ll
@@ -0,0 +1,13 @@
+; REQUIRES: asserts
+; RUN: opt < %s -passes=instcombine -verify-pgo-flow \
+; RUN: -debug-only=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+;
+; No ProfileSummary: walk the function, then skip InstrProf use-phase checks.
+
+; CHECK: *** PGO Flow Verification After InstCombinePass ***{{$}}
+; CHECK: PGOFlowVerifier: skip 'f' (no InstrProf use-phase summary)
+
+define i32 @f(i32 %x) {
+ %a = add i32 %x, 0
+ ret i32 %a
+}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-sample-profile.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-sample-profile.ll
new file mode 100644
index 0000000000000..a5eff400efb9e
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-sample-profile.ll
@@ -0,0 +1,27 @@
+; REQUIRES: asserts
+; RUN: opt < %s -passes=instcombine -verify-pgo-flow \
+; RUN: -debug-only=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+;
+; SampleProfile summary is not InstrProf use-phase metadata.
+
+; CHECK: *** PGO Flow Verification After InstCombinePass ***{{$}}
+; CHECK: PGOFlowVerifier: skip 'f' (no InstrProf use-phase summary)
+
+define i32 @f(i32 %x) {
+ %a = add i32 %x, 0
+ ret i32 %a
+}
+
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"SampleProfile"}
+!4 = !{!"TotalCount", i64 1}
+!5 = !{!"MaxCount", i64 1}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1}
+!8 = !{!"NumCounts", i64 1}
+!9 = !{!"NumFunctions", i64 1}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12}
+!12 = !{i32 10000, i64 1, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-standalone-pass.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-standalone-pass.ll
new file mode 100644
index 0000000000000..372becf657719
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-standalone-pass.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=MODULE
+; RUN: opt < %s -passes='function(verify-pgo-flow)' -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=FUNCTION
+; RUN: opt < %s -passes=instcombine -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=NOTRUN --allow-empty
+; RUN: opt < %s -passes=verify-pgo-flow \
+; RUN: -verify-pgo-flow-print-diagnostics=false -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=QUIET --allow-empty
+;
+; Named pipeline pass; does not need -verify-pgo-flow and does not change IR.
+
+; MODULE: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; MODULE-NOT: (Skipped)
+
+; FUNCTION: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; FUNCTION-NOT: (Skipped)
+
+; NOTRUN-NOT: PGO Flow Verification
+
+; QUIET-NOT: PGO Flow Verification
+
+define i32 @f(i32 %x) {
+ ret i32 %x
+}
>From 06bdc1338386ab11ddec6d39447f5f8fe45d49dd Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:03:23 +0530
Subject: [PATCH 02/13] [PGOFlowVerify] Check in-function block flow
conservation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
When InstrProf summary is present, report basic blocks whose known
With InstrProf, for each live block:
- Known count-type in equals known count-type out.
- Hints and unknown edges are not counts.
- Leftover MD on unreachable or 0-in blocks is not flow.
- Loops: apply the header’s count-type weights before the backedge
count is known, otherwise the cycle never closes.
---
.../llvm/Transforms/IPO/PGOFlowVerify.h | 17 +
llvm/lib/Passes/PassBuilder.cpp | 32 ++
llvm/lib/Passes/PassRegistry.def | 1 +
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 309 +++++++++++++++++-
.../verify-pgo-flow-block-frequency.ll | 73 +++++
...verify-pgo-flow-expected-branch-weights.ll | 36 ++
.../verify-pgo-flow-loop-flow.ll | 60 ++++
.../verify-pgo-flow-switch-multi-edge.ll | 38 +++
...y-pgo-flow-unreachable-leftover-weights.ll | 143 ++++++++
9 files changed, 695 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-block-frequency.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-expected-branch-weights.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-loop-flow.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-switch-multi-edge.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unreachable-leftover-weights.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
index 9704f5e3b2b98..bd1e67888a765 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -16,6 +16,8 @@
#ifndef LLVM_TRANSFORMS_IPO_PGOFLOWVERIFY_H
#define LLVM_TRANSFORMS_IPO_PGOFLOWVERIFY_H
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/IR/IRUnitRef.h"
@@ -23,6 +25,7 @@
#include "llvm/Support/Compiler.h"
namespace llvm {
+class BasicBlock;
class Function;
class Loop;
class Module;
@@ -31,17 +34,31 @@ class PassInstrumentationCallbacks;
/// Walk IR after transforms so InstrProf use-phase flow checks can run.
class PGOFlowVerifier {
public:
+ struct BlockFreqInfo {
+ unsigned NumUnknownIn = 0;
+ unsigned NumUnknownOut = 0;
+ uint64_t SumIn = 0;
+ uint64_t SumOut = 0;
+ };
+ using AllBlockFreqInfo = MapVector<const BasicBlock *, BlockFreqInfo>;
+
/// True when `-verify-pgo-flow` is set.
LLVM_ABI static bool isHookEnabled();
LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC);
LLVM_ABI void runAfterPass(StringRef PassID, IRUnitRef IR);
private:
+ void invalidateFunctionFrequencyCache(IRUnitRef IR);
void runAfterPass(const Module *M);
void runAfterPass(const Function *F);
void runAfterPass(const LazyCallGraph::SCC *C);
void runAfterPass(const Loop *L);
bool hasInstrProfUseSummary(const Module *M) const;
+
+ void computeBlockFrequencies(const Function *F);
+ void validateBlockFrequencies(const Function *F);
+
+ DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
};
/// Pipeline pass that runs the same walk as the `-verify-pgo-flow` hook.
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index fc4c54dad9666..5f3860a33fadd 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -211,6 +211,7 @@
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/SafepointIRVerifier.h"
#include "llvm/IR/Verifier.h"
#include "llvm/IRPrinter/IRPrintingPasses.h"
@@ -534,6 +535,37 @@ class TriggerVerifierErrorPass
static StringRef name() { return "TriggerVerifierErrorPass"; }
};
+// Test-only: corrupt branch_weights so in-function flow no longer adds up.
+// Used to check that `-verify-pgo-flow` runs after a transforming pass.
+// DO NOT USE THIS EXCEPT FOR TESTING!
+class BreakPGOFlowBranchWeightsPass
+ : public OptionalPassInfoMixin<BreakPGOFlowBranchWeightsPass> {
+public:
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
+ if (F.isDeclaration())
+ return PreservedAnalyses::all();
+
+ for (BasicBlock &BB : F) {
+ Instruction *Term = BB.getTerminator();
+ if (!Term)
+ continue;
+ SmallVector<uint32_t, 8> Weights;
+ if (!extractBranchWeights(*Term, Weights) ||
+ Weights.size() != Term->getNumSuccessors() || Weights.empty())
+ continue;
+ if (Weights[0] > 0)
+ --Weights[0];
+ else
+ ++Weights[0];
+ setBranchWeights(*Term, Weights, /*IsExpected=*/false);
+ return PreservedAnalyses::none();
+ }
+ return PreservedAnalyses::all();
+ }
+
+ static StringRef name() { return "BreakPGOFlowBranchWeightsPass"; }
+};
+
// A pass requires all MachineFunctionProperties.
// DO NOT USE THIS EXCEPT FOR TESTING!
class RequireAllMachineFunctionPropertiesPass
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 216e39d1830c7..7983357b3ea16 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -418,6 +418,7 @@ FUNCTION_PASS("assume-simplify", AssumeSimplifyPass())
FUNCTION_PASS("atomic-expand", AtomicExpandPass(*TM))
FUNCTION_PASS("bdce", BDCEPass())
FUNCTION_PASS("break-crit-edges", BreakCriticalEdgesPass())
+FUNCTION_PASS("break-pgo-flow-branch-weights", BreakPGOFlowBranchWeightsPass())
FUNCTION_PASS("callsite-splitting", CallSiteSplittingPass())
FUNCTION_PASS("cfguard", CFGuardPass())
FUNCTION_PASS("chr", ControlHeightReductionPass())
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
index d24d9aa2d94ed..d4543a5a5ade1 100644
--- a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -12,16 +12,25 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/IPO/PGOFlowVerify.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassInstrumentation.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
+#include <numeric>
#include <vector>
using namespace llvm;
@@ -36,9 +45,9 @@ static cl::opt<bool> VerifyPGOFlowPrintDiagnostics(
"verify-pgo-flow-print-diagnostics", cl::init(true), cl::Hidden,
cl::desc("Print verify-pgo-flow banners and findings to stderr"));
-static cl::opt<bool> VerifyPGOFlowFatal(
- "verify-pgo-flow-fatal", cl::init(false), cl::Hidden,
- cl::desc("Abort after a flow-check finding (no-op until checks land)"));
+static cl::opt<bool>
+ VerifyPGOFlowFatal("verify-pgo-flow-fatal", cl::init(false), cl::Hidden,
+ cl::desc("Abort after a flow-check finding"));
static void printVerifyBanner(StringRef PassName, bool Skipped) {
if (!VerifyPGOFlowPrintDiagnostics)
@@ -49,6 +58,20 @@ static void printVerifyBanner(StringRef PassName, bool Skipped) {
bool PGOFlowVerifier::isHookEnabled() { return VerifyPGOFlow; }
+static void emitPGOFlowDiagnostic(const Function *F, StringRef RemarkName,
+ const Twine &Msg) {
+ if (!F)
+ return;
+ std::string Text = Msg.str();
+ if (VerifyPGOFlowPrintDiagnostics)
+ errs() << "PGOFlowVerify[" << RemarkName << "] " << F->getName() << ": "
+ << Text << "\n";
+ if (VerifyPGOFlowFatal)
+ report_fatal_error(Twine("PGOFlowVerify[") + RemarkName + "] " +
+ F->getName() + ": " + Text,
+ /*gen_crash_diag=*/false);
+}
+
void PGOFlowVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
if (!VerifyPGOFlow) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: hook not registered "
@@ -88,7 +111,37 @@ void PGOFlowVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
});
}
+void PGOFlowVerifier::invalidateFunctionFrequencyCache(IRUnitRef IR) {
+ if (isa<Module>(IR)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: clear block-freq cache (module)\n");
+ FunctionBlockFreqInfoCache.clear();
+ return;
+ }
+ if (const auto *F = dyn_cast<Function>(IR)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: drop block-freq cache for '"
+ << F->getName() << "'\n");
+ FunctionBlockFreqInfoCache.erase(F);
+ return;
+ }
+ if (const auto *C = dyn_cast<LazyCallGraph::SCC>(IR)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: drop block-freq cache for SCC\n");
+ for (const LazyCallGraph::Node &N : *C)
+ FunctionBlockFreqInfoCache.erase(&N.getFunction());
+ return;
+ }
+ if (const auto *L = dyn_cast<Loop>(IR)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: drop block-freq cache for loop\n");
+ if (L->getHeader())
+ FunctionBlockFreqInfoCache.erase(L->getHeader()->getParent());
+ return;
+ }
+ LLVM_DEBUG(
+ dbgs() << "PGOFlowVerifier: clear block-freq cache (unhandled IR)\n");
+ FunctionBlockFreqInfoCache.clear();
+}
+
void PGOFlowVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
+ invalidateFunctionFrequencyCache(IR);
if (const auto *M = dyn_cast<Module>(IR)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: module IR after " << PassID << "\n");
runAfterPass(M);
@@ -111,6 +164,11 @@ void PGOFlowVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
void PGOFlowVerifier::runAfterPass(const Module *M) {
if (!M)
return;
+ if (!hasInstrProfUseSummary(M)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip module '" << M->getName()
+ << "' (no InstrProf use-phase summary)\n");
+ return;
+ }
for (const Function &F : *M)
runAfterPass(&F);
}
@@ -123,17 +181,8 @@ void PGOFlowVerifier::runAfterPass(const Function *F) {
<< "' (no InstrProf use-phase summary)\n");
return;
}
-}
-
-bool PGOFlowVerifier::hasInstrProfUseSummary(const Module *M) const {
- if (!M)
- return false;
- Metadata *SummaryMD = M->getProfileSummary(/*IsCS=*/false);
- if (!SummaryMD)
- return false;
- std::unique_ptr<ProfileSummary> PS(ProfileSummary::getFromMD(SummaryMD));
- return PS && (PS->getKind() == ProfileSummary::PSK_Instr ||
- PS->getKind() == ProfileSummary::PSK_CSInstr);
+ computeBlockFrequencies(F);
+ validateBlockFrequencies(F);
}
void PGOFlowVerifier::runAfterPass(const LazyCallGraph::SCC *C) {
@@ -149,6 +198,238 @@ void PGOFlowVerifier::runAfterPass(const Loop *L) {
runAfterPass(L->getHeader()->getParent());
}
+void PGOFlowVerifier::computeBlockFrequencies(const Function *F) {
+ if (!F)
+ return;
+
+ AllBlockFreqInfo AllFreqInfo;
+ for (const BasicBlock &BB : *F) {
+ AllFreqInfo[&BB].NumUnknownIn = pred_size(&BB);
+ AllFreqInfo[&BB].NumUnknownOut = succ_size(&BB);
+ }
+
+ AllFreqInfo[&F->getEntryBlock()].NumUnknownIn = 1;
+ if (std::optional<uint64_t> Count = F->getEntryCount()) {
+ AllFreqInfo[&F->getEntryBlock()].SumIn = *Count;
+ AllFreqInfo[&F->getEntryBlock()].NumUnknownIn = 0;
+ if (*Count == 0) {
+ for (const BasicBlock &BB : *F) {
+ AllFreqInfo[&BB].NumUnknownIn = 0;
+ AllFreqInfo[&BB].SumIn = 0;
+ AllFreqInfo[&BB].NumUnknownOut = 0;
+ AllFreqInfo[&BB].SumOut = 0;
+ }
+ FunctionBlockFreqInfoCache[F] = std::move(AllFreqInfo);
+ return;
+ } else if (const Instruction *EntryTerm =
+ F->getEntryBlock().getTerminator();
+ EntryTerm && EntryTerm->getNumSuccessors() == 0) {
+ AllFreqInfo[&F->getEntryBlock()].SumOut = *Count;
+ AllFreqInfo[&F->getEntryBlock()].NumUnknownOut = 0;
+ }
+ }
+
+ SmallVector<const BasicBlock *, 16> Worklist;
+ SmallPtrSet<const BasicBlock *, 16> InWorklist;
+ auto Enqueue = [&](const BasicBlock *BB) {
+ if (BB && InWorklist.insert(BB).second)
+ Worklist.push_back(BB);
+ };
+
+ // Unreachable blocks (including cyclic leftover SCCs) are not live flow.
+ SmallPtrSet<const BasicBlock *, 16> Reachable;
+ SmallVector<const BasicBlock *, 16> ReachWork;
+ ReachWork.push_back(&F->getEntryBlock());
+ Reachable.insert(&F->getEntryBlock());
+ while (!ReachWork.empty()) {
+ const BasicBlock *BB = ReachWork.pop_back_val();
+ const Instruction *Term = BB->getTerminator();
+ if (!Term)
+ continue;
+ for (unsigned I = 0, E = Term->getNumSuccessors(); I < E; ++I) {
+ const BasicBlock *Succ = Term->getSuccessor(I);
+ if (Reachable.insert(Succ).second)
+ ReachWork.push_back(Succ);
+ }
+ }
+
+ auto HasCountTypeOutgoing = [&](const BasicBlock *BB) {
+ const Instruction *Term = BB->getTerminator();
+ if (!Term || Term->getNumSuccessors() == 0)
+ return false;
+ if (hasBranchWeightOrigin(*Term) ||
+ hasExplicitlyUnknownBranchWeights(*Term))
+ return false;
+ MDNode *WeightMD = getValidBranchWeightMDNode(*Term);
+ if (!WeightMD)
+ return false;
+ SmallVector<uint64_t, 8> Weights;
+ extractFromBranchWeightMD64(WeightMD, Weights);
+ return Weights.size() == Term->getNumSuccessors();
+ };
+
+ auto ShouldProcess = [&](const BasicBlock *BB) {
+ if (!BB || !Reachable.contains(BB))
+ return false;
+ const BlockFreqInfo &Info = AllFreqInfo[BB];
+ if (Info.NumUnknownIn == 0)
+ return true;
+ // Live flow already credited, but a backedge is still unknown. Apply
+ // count-type outs now. Do not early-walk unweighted blocks or leftover
+ // !prof on a 0-in path.
+ return Info.SumIn > 0 && HasCountTypeOutgoing(BB);
+ };
+
+ auto ReleaseEdge = [&](const BasicBlock *Succ, uint64_t Add) {
+ if (!Succ)
+ return;
+ BlockFreqInfo &SuccInfo = AllFreqInfo[Succ];
+ if (SuccInfo.NumUnknownIn > 0)
+ SuccInfo.NumUnknownIn--;
+ if (Add)
+ SuccInfo.SumIn += Add;
+ if (ShouldProcess(Succ))
+ Enqueue(Succ);
+ };
+
+ for (const BasicBlock &BB : *F) {
+ if (Reachable.contains(&BB))
+ continue;
+ BlockFreqInfo &Info = AllFreqInfo[&BB];
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip leftover branch weights on "
+ << "unreachable block " << BB.getName() << " in '"
+ << F->getName() << "'\n");
+ Info.NumUnknownIn = 0;
+ Info.SumIn = 0;
+ Info.NumUnknownOut = 0;
+ Info.SumOut = 0;
+ const Instruction *Term = BB.getTerminator();
+ if (!Term)
+ continue;
+ // Drop dead preds from live successors so conservation can still run.
+ for (unsigned I = 0, E = Term->getNumSuccessors(); I < E; ++I) {
+ const BasicBlock *Succ = Term->getSuccessor(I);
+ if (Succ && Reachable.contains(Succ))
+ ReleaseEdge(Succ, 0);
+ }
+ }
+
+ auto ProcessResolved = [&](const BasicBlock *BB) {
+ const Instruction *Term = BB->getTerminator();
+ if (!Term)
+ return;
+ BlockFreqInfo &Info = AllFreqInfo[BB];
+ if (isa<ReturnInst>(Term)) {
+ if (Info.NumUnknownIn != 0)
+ return;
+ Info.SumOut = Info.SumIn;
+ Info.NumUnknownOut = 0;
+ return;
+ }
+ // Known-zero incoming: do not extract or apply leftover branch weights.
+ if (Info.NumUnknownIn == 0 && Info.SumIn == 0) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip leftover branch weights on "
+ << "unreachable/zero-count block " << BB->getName()
+ << " in '" << F->getName() << "'\n");
+ Info.NumUnknownOut = 0;
+ Info.SumOut = 0;
+ for (unsigned I = 0, E = Term->getNumSuccessors(); I < E; ++I)
+ ReleaseEdge(Term->getSuccessor(I), 0);
+ return;
+ }
+ // Count-type InstrProf branch_weights only. !"expected" (llvm.expect)
+ // origins are probabilities, not execution counts. Explicitly unknown
+ // weights are not counts either. Leave NumUnknownIn/NumUnknownOut as-is.
+ if (hasBranchWeightOrigin(*Term) ||
+ hasExplicitlyUnknownBranchWeights(*Term)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip non-count branch weights in '"
+ << F->getName() << "' block " << BB->getName() << "\n");
+ return;
+ }
+ SmallVector<uint32_t, 8> Weights32;
+ if (extractBranchWeights(*Term, Weights32) &&
+ Weights32.size() == Term->getNumSuccessors()) {
+ // Outs already closed (weights applied while a backedge was unknown).
+ if (Info.NumUnknownOut == 0)
+ return;
+ // No live flow yet.
+ if (Info.SumIn == 0)
+ return;
+ SmallVector<uint64_t, 8> Weights(Weights32.begin(), Weights32.end());
+ Info.NumUnknownOut = 0;
+ Info.SumOut =
+ std::accumulate(Weights.begin(), Weights.end(), uint64_t(0));
+ for (unsigned I = 0, E = Term->getNumSuccessors(); I < E; ++I)
+ ReleaseEdge(Term->getSuccessor(I), Weights[I]);
+ return;
+ }
+ if (Info.NumUnknownIn != 0)
+ return;
+ if (Info.NumUnknownOut == 1 && Term->getNumSuccessors() == 1) {
+ ReleaseEdge(Term->getSuccessor(0), Info.SumIn);
+ Info.NumUnknownOut = 0;
+ Info.SumOut = Info.SumIn;
+ }
+ };
+
+ // Count-type weights can close outs before every pred is known (live
+ // loops). Unweighted copy and 0-in leftover skip still wait for pred close
+ // so BB-list order cannot apply !prof on a child listed before a 0-edge.
+ for (const BasicBlock &BB : *F)
+ if (ShouldProcess(&BB))
+ Enqueue(&BB);
+ while (!Worklist.empty()) {
+ const BasicBlock *BB = Worklist.pop_back_val();
+ ProcessResolved(BB);
+ }
+
+ FunctionBlockFreqInfoCache[F] = std::move(AllFreqInfo);
+}
+
+void PGOFlowVerifier::validateBlockFrequencies(const Function *F) {
+ if (!F)
+ return;
+ auto CachedIt = FunctionBlockFreqInfoCache.find(F);
+ if (CachedIt == FunctionBlockFreqInfoCache.end()) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: no block-freq cache for '"
+ << F->getName() << "'\n");
+ return;
+ }
+
+ const AllBlockFreqInfo &AllFreqInfo = CachedIt->second;
+ for (const BasicBlock &BB : *F) {
+ const Instruction *Term = BB.getTerminator();
+ if (!Term || Term->getNumSuccessors() == 0)
+ continue;
+ auto It = AllFreqInfo.find(&BB);
+ if (It == AllFreqInfo.end())
+ continue;
+ const BlockFreqInfo &Info = It->second;
+ if (Info.NumUnknownIn == 0 && Info.NumUnknownOut == 0 &&
+ Info.SumIn != Info.SumOut)
+ emitPGOFlowDiagnostic(F, "BlockFrequencyMismatch",
+ Twine("block ") + BB.getName() +
+ ": incoming=" + Twine(Info.SumIn) +
+ " vs outgoing=" + Twine(Info.SumOut));
+ else if (Info.NumUnknownIn != 0 || Info.NumUnknownOut != 0)
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: unknown edges in '" << F->getName()
+ << "' block " << BB.getName()
+ << " in=" << Info.NumUnknownIn
+ << " out=" << Info.NumUnknownOut << "\n");
+ }
+}
+
+bool PGOFlowVerifier::hasInstrProfUseSummary(const Module *M) const {
+ if (!M)
+ return false;
+ Metadata *SummaryMD = M->getProfileSummary(/*IsCS=*/false);
+ if (!SummaryMD)
+ return false;
+ std::unique_ptr<ProfileSummary> PS(ProfileSummary::getFromMD(SummaryMD));
+ return PS && (PS->getKind() == ProfileSummary::PSK_Instr ||
+ PS->getKind() == ProfileSummary::PSK_CSInstr);
+}
+
PreservedAnalyses PGOFlowVerifierPass::run(Module &M,
ModuleAnalysisManager &MAM) {
(void)MAM;
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-block-frequency.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-block-frequency.ll
new file mode 100644
index 0000000000000..11b1d7e7e0941
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-block-frequency.ll
@@ -0,0 +1,73 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=PASS
+; RUN: opt < %s -passes=break-pgo-flow-branch-weights -verify-pgo-flow \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=HOOK
+; RUN: opt < %s -passes='break-pgo-flow-branch-weights,verify-pgo-flow' \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=PIPE
+; RUN: not opt < %s -passes=verify-pgo-flow -verify-pgo-flow-fatal \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=FATAL
+;
+; @ok is consistent; @bad is not. The standalone pass reports only @bad.
+; break-pgo-flow-branch-weights corrupts weights. HOOK uses the post-pass
+; `-verify-pgo-flow` flag; PIPE puts `verify-pgo-flow` in the pipeline
+; instead (no `-verify-pgo-flow`). Needs an InstrProf ProfileSummary.
+
+; PASS: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; PASS-NOT: PGOFlowVerify[BlockFrequencyMismatch] ok:
+; PASS: PGOFlowVerify[BlockFrequencyMismatch] bad: block entry: incoming=10 vs outgoing=9
+; PASS-NOT: PGOFlowVerify[BlockFrequencyMismatch] ok:
+
+; HOOK: *** PGO Flow Verification After BreakPGOFlowBranchWeightsPass ***{{$}}
+; HOOK: PGOFlowVerify[BlockFrequencyMismatch] ok: block entry: incoming=10 vs outgoing=9
+; HOOK: *** PGO Flow Verification After BreakPGOFlowBranchWeightsPass ***
+; HOOK: PGOFlowVerify[BlockFrequencyMismatch] bad: block entry: incoming=10 vs outgoing=8
+
+; PIPE-NOT: BreakPGOFlowBranchWeightsPass
+; PIPE: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; PIPE: PGOFlowVerify[BlockFrequencyMismatch] ok: block entry: incoming=10 vs outgoing=9
+; PIPE: PGOFlowVerify[BlockFrequencyMismatch] bad: block entry: incoming=10 vs outgoing=8
+; PIPE-NOT: *** PGO Flow Verification After BreakPGOFlowBranchWeightsPass
+
+; FATAL: PGOFlowVerify[BlockFrequencyMismatch]
+
+define i32 @ok(i32 %x) !prof !0 {
+entry:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !1
+
+then:
+ ret i32 1
+
+else:
+ ret i32 0
+}
+
+define i32 @bad(i32 %x) !prof !0 {
+entry:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !2
+
+then:
+ ret i32 1
+
+else:
+ ret i32 0
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 7, i32 3}
+!2 = !{!"branch_weights", i32 7, i32 2}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 10}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 7}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 2}
+!18 = !{!"NumFunctions", i64 2}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-expected-branch-weights.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-expected-branch-weights.ll
new file mode 100644
index 0000000000000..6e964db1f3ba1
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-expected-branch-weights.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+;
+; !"expected" branch_weights are probabilities (llvm.expect), not InstrProf
+; counts. Do not report BlockFrequencyMismatch for well-formed expected MD.
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+
+define i32 @expected_br(i32 %x) !prof !0 {
+entry:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !1
+
+then:
+ ret i32 1
+
+else:
+ ret i32 0
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", !"expected", i32 1, i32 2000}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 10}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 2}
+!18 = !{!"NumFunctions", i64 1}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-loop-flow.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-loop-flow.ll
new file mode 100644
index 0000000000000..61c06966995c6
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-loop-flow.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+; RUN: not opt < %s -passes=verify-pgo-flow -verify-pgo-flow-fatal \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=FATAL
+;
+; Count-type branch_weights on a live loop header must be applied before the
+; backedge pred closes. @ok_loop conserves (entry 10 + backedge 90 = 100).
+; @bad_loop does not (outgoing 95 vs incoming 100).
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch] ok_loop:
+; CHECK: PGOFlowVerify[BlockFrequencyMismatch] bad_loop: block header: incoming=100 vs outgoing=95
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch] ok_loop:
+
+; FATAL: PGOFlowVerify[BlockFrequencyMismatch] bad_loop:
+
+define void @ok_loop(i1 %c) !prof !0 {
+entry:
+ br label %header
+
+header:
+ br i1 %c, label %latch, label %exit, !prof !1
+
+latch:
+ br label %header
+
+exit:
+ ret void
+}
+
+define void @bad_loop(i1 %c) !prof !0 {
+entry:
+ br label %header
+
+header:
+ br i1 %c, label %latch, label %exit, !prof !2
+
+latch:
+ br label %header
+
+exit:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 90, i32 10}
+!2 = !{!"branch_weights", i32 90, i32 5}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 10}
+!14 = !{!"MaxCount", i64 90}
+!15 = !{!"MaxInternalCount", i64 90}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 4}
+!18 = !{!"NumFunctions", i64 2}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 90, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-switch-multi-edge.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-switch-multi-edge.ll
new file mode 100644
index 0000000000000..e50ff7e83f110
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-switch-multi-edge.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+;
+; Two switch cases share a successor. pred_size counts Uses, not unique
+; predecessors, so both edges are unknown-ins and conservation holds.
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+
+define i32 @switch_multi(i32 %x) !prof !0 {
+entry:
+ switch i32 %x, label %def [
+ i32 0, label %hot
+ i32 1, label %hot
+ ], !prof !1
+
+hot:
+ ret i32 1
+
+def:
+ ret i32 0
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 1, i32 6, i32 3}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 10}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 2}
+!18 = !{!"NumFunctions", i64 1}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unreachable-leftover-weights.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unreachable-leftover-weights.ll
new file mode 100644
index 0000000000000..2e9becb07ed87
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unreachable-leftover-weights.ll
@@ -0,0 +1,143 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+; RUN: not opt < %s -passes=verify-pgo-flow -verify-pgo-flow-fatal \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=FATAL
+;
+; Unreachable blocks can keep stale count-type branch_weights after CFG
+; edits. Those weights are not live flow, do not report BlockFrequencyMismatch.
+; Unreachable SCCs (every block has pred_size > 0) and leftover weights on a
+; zero function_entry_count terminator are the same class of leftover MD.
+;
+; @order: a 0-weight successor listed after a child that still has leftover
+; !prof. Do not apply that MD in raw BB-list order.
+; @dead_pred: %live has preds {entry, %dead}, %dead is unreachable. Releasing
+; the dead pred must still conservation-check %live.
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch] leftover:
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch] leftover_cycle:
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch] zero_entry:
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch] order:
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch] dead_pred:
+; CHECK: PGOFlowVerify[BlockFrequencyMismatch] dead_pred_bad: block live: incoming=10 vs outgoing=9
+
+; FATAL: PGOFlowVerify[BlockFrequencyMismatch] dead_pred_bad:
+
+define i32 @leftover(i32 %x) !prof !0 {
+entry:
+ ret i32 0
+
+dead:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !1
+
+then:
+ ret i32 1
+
+else:
+ %c2 = icmp eq i32 %x, 0
+ br i1 %c2, label %join, label %other, !prof !2
+
+other:
+ ret i32 3
+
+join:
+ ret i32 2
+}
+
+define i32 @leftover_cycle(i32 %x) !prof !0 {
+entry:
+ ret i32 0
+
+dead1:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %dead2, label %dead1, !prof !1
+
+dead2:
+ br label %dead1
+}
+
+define void @zero_entry(i1 %c) !prof !3 {
+entry:
+ br i1 %c, label %a, label %b, !prof !1
+
+a:
+ ret void
+
+b:
+ ret void
+}
+
+define i32 @order(i1 %c) !prof !0 {
+entry:
+ br i1 %c, label %live, label %zero, !prof !4
+
+child:
+ br i1 %c, label %child.a, label %child.b, !prof !1
+
+child.a:
+ ret i32 1
+
+child.b:
+ ret i32 2
+
+zero:
+ br i1 %c, label %child, label %child, !prof !1
+
+live:
+ ret i32 0
+}
+
+define i32 @dead_pred(i1 %c) !prof !0 {
+entry:
+ br label %live
+
+dead:
+ br i1 %c, label %live, label %live, !prof !1
+
+live:
+ br i1 %c, label %a, label %b, !prof !1
+
+a:
+ ret i32 0
+
+b:
+ ret i32 1
+}
+
+define i32 @dead_pred_bad(i1 %c) !prof !0 {
+entry:
+ br label %live
+
+dead:
+ br i1 %c, label %live, label %live, !prof !1
+
+live:
+ br i1 %c, label %a, label %b, !prof !6
+
+a:
+ ret i32 0
+
+b:
+ ret i32 1
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 7, i32 3}
+!2 = !{!"branch_weights", i32 9, i32 1}
+!3 = !{!"function_entry_count", i64 0}
+!4 = !{!"branch_weights", i32 10, i32 0}
+!6 = !{!"branch_weights", i32 9, i32 0}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 10}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 7}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 2}
+!18 = !{!"NumFunctions", i64 6}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
>From 39e200b87b9987877e27ad21cc3acf4f32ecb95e Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:03:41 +0530
Subject: [PATCH 03/13] [PGOFlowVerify] Check entry count against direct caller
sum
Report EntryCountMismatch when the known direct caller-sum exceeds
the callee's function_entry_count. Unknown extra sites do not hide
that overcount. Skip when there is no visible site, when known sum
does not exceed entry and some site is unknown, and when the callee
is recursive. Module walk only. Do not credit leftover !prof on a
dead caller block.
---
.../llvm/Transforms/IPO/PGOFlowVerify.h | 2 +
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 151 +++++++++++++++++-
.../verify-pgo-flow-dead-callsite.ll | 55 +++++++
.../verify-pgo-flow-entry-count.ll | 88 ++++++++++
.../verify-pgo-flow-folded-vp-callsite.ll | 76 +++++++++
...ify-pgo-flow-unknown-callsite-overcount.ll | 43 +++++
.../verify-pgo-flow-unweighted-callsite.ll | 38 +++++
7 files changed, 450 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dead-callsite.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-entry-count.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-folded-vp-callsite.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unknown-callsite-overcount.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unweighted-callsite.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
index bd1e67888a765..a31385b4e9296 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -57,6 +57,8 @@ class PGOFlowVerifier {
void computeBlockFrequencies(const Function *F);
void validateBlockFrequencies(const Function *F);
+ void validateEntryCountAgainstCallerSum(const Function *F);
+ const AllBlockFreqInfo *getCachedBlockFreqInfo(const Function *F) const;
DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
};
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
index d4543a5a5ade1..f5e9e39a57026 100644
--- a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -18,8 +18,13 @@
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
+#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalAlias.h"
+#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassInstrumentation.h"
#include "llvm/IR/PassManager.h"
@@ -28,6 +33,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <numeric>
@@ -169,12 +175,23 @@ void PGOFlowVerifier::runAfterPass(const Module *M) {
<< "' (no InstrProf use-phase summary)\n");
return;
}
- for (const Function &F : *M)
- runAfterPass(&F);
+ // Fill the per-function cache first so callee checks can see caller blocks
+ // no matter which order functions appear in the module.
+ for (const Function &F : *M) {
+ if (F.isDeclaration())
+ continue;
+ computeBlockFrequencies(&F);
+ validateBlockFrequencies(&F);
+ }
+ for (const Function &F : *M) {
+ if (F.isDeclaration())
+ continue;
+ validateEntryCountAgainstCallerSum(&F);
+ }
}
void PGOFlowVerifier::runAfterPass(const Function *F) {
- if (!F || F->isDeclaration())
+ if (!F || F->isDeclaration() || !F->getParent())
return;
if (!hasInstrProfUseSummary(F->getParent())) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip '" << F->getName()
@@ -183,6 +200,8 @@ void PGOFlowVerifier::runAfterPass(const Function *F) {
}
computeBlockFrequencies(F);
validateBlockFrequencies(F);
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '" << F->getName()
+ << "' (function-unit walk; need module-wide caller BFI)\n");
}
void PGOFlowVerifier::runAfterPass(const LazyCallGraph::SCC *C) {
@@ -419,6 +438,132 @@ void PGOFlowVerifier::validateBlockFrequencies(const Function *F) {
}
}
+const PGOFlowVerifier::AllBlockFreqInfo *
+PGOFlowVerifier::getCachedBlockFreqInfo(const Function *F) const {
+ if (!F)
+ return nullptr;
+ auto It = FunctionBlockFreqInfoCache.find(F);
+ if (It == FunctionBlockFreqInfoCache.end())
+ return nullptr;
+ return &It->second;
+}
+
+void PGOFlowVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
+ if (!F)
+ return;
+
+ std::optional<uint64_t> MaybeEntryCount = F->getEntryCount();
+ if (!MaybeEntryCount) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
+ << F->getName() << "' (no entry count)\n");
+ return;
+ }
+ uint64_t EntryCount = *MaybeEntryCount;
+
+ uint64_t Sum = 0;
+ bool IsRecursive = false;
+ bool HasAnyDirectCallsite = false;
+ bool HasUnknownCallsiteCount = false;
+
+ auto ConsiderCallsite = [&](const CallBase *CB) {
+ const BasicBlock *BB = CB->getParent();
+ if (!BB)
+ return;
+ const Function *CallerFunc = BB->getParent();
+ if (!CallerFunc)
+ return;
+ if (CallerFunc == F)
+ IsRecursive = true;
+
+ const AllBlockFreqInfo *CallerFreq = getCachedBlockFreqInfo(CallerFunc);
+ if (!CallerFreq) {
+ HasUnknownCallsiteCount = true;
+ return;
+ }
+ auto CallerBBIt = CallerFreq->find(BB);
+ if (CallerBBIt == CallerFreq->end() ||
+ CallerBBIt->second.NumUnknownIn != 0) {
+ HasUnknownCallsiteCount = true;
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: unknown caller-block flow for '"
+ << F->getName() << "' from '" << CallerFunc->getName()
+ << "'\n");
+ return;
+ }
+ bool NonzeroEntry = BB == &CallerFunc->getEntryBlock() &&
+ CallerFunc->getEntryCount() &&
+ *CallerFunc->getEntryCount() != 0;
+ if (CallerBBIt->second.SumIn == 0 && !NonzeroEntry) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip dead-block callsite for '"
+ << F->getName() << "' from '" << CallerFunc->getName()
+ << "'\n");
+ return;
+ }
+
+ uint64_t CallsiteCount = 0;
+ MDNode *MD = CB->getMetadata(LLVMContext::MD_prof);
+ // Direct sites only credit count-type branch_weights. VP aggregates
+ // every target; llvm.expect and unknown are not InstrProf counts.
+ if (isValueProfileMD(MD) || hasBranchWeightOrigin(MD) ||
+ (MD && isExplicitlyUnknownProfileMetadata(*MD)) ||
+ !extractProfTotalWeight(MD, CallsiteCount)) {
+ HasUnknownCallsiteCount = true;
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: unknown callsite weight for '"
+ << F->getName() << "' from '" << CallerFunc->getName()
+ << "' block " << BB->getName() << "\n");
+ return;
+ }
+ HasAnyDirectCallsite = true;
+ Sum = SaturatingAdd(Sum, CallsiteCount);
+ };
+
+ SmallVector<const User *, 8> Worklist(F->user_begin(), F->user_end());
+ SmallPtrSet<const User *, 16> Visited;
+ while (!Worklist.empty()) {
+ const User *U = Worklist.pop_back_val();
+ if (!Visited.insert(U).second)
+ continue;
+ if (const auto *CB = dyn_cast<CallBase>(U)) {
+ if (CB->getCalledOperand()->stripPointerCastsAndAliases() != F)
+ continue;
+ ConsiderCallsite(CB);
+ continue;
+ }
+ if (isa<ConstantExpr>(U) || isa<GlobalAlias>(U)) {
+ for (const User *UU : U->users())
+ Worklist.push_back(UU);
+ continue;
+ }
+ }
+
+ if (!HasAnyDirectCallsite) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
+ << F->getName() << "' (no direct callsite)\n");
+ return;
+ }
+ if (IsRecursive) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
+ << F->getName() << "' (recursive)\n");
+ return;
+ }
+ // Known Sum is a lower bound. Unknown sites can hide undercount, but not
+ // a definite overcount.
+ if (HasUnknownCallsiteCount && Sum <= EntryCount) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
+ << F->getName() << "' (unknown callsite weight)\n");
+ return;
+ }
+ if (Sum <= EntryCount) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
+ << F->getName() << "' (caller-sum=" << Sum
+ << " <= entry=" << EntryCount << ")\n");
+ return;
+ }
+
+ emitPGOFlowDiagnostic(F, "EntryCountMismatch",
+ Twine("entry=") + Twine(EntryCount) +
+ " vs caller-sum=" + Twine(Sum));
+}
+
bool PGOFlowVerifier::hasInstrProfUseSummary(const Module *M) const {
if (!M)
return false;
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dead-callsite.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dead-callsite.ll
new file mode 100644
index 0000000000000..28dc1d293818f
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dead-callsite.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+;
+; Leftover call !prof on an unreachable block is not live flow. Do not add
+; it into the callee's caller-sum (would be entry=1 vs caller-sum=10).
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK-NOT: PGOFlowVerify[EntryCountMismatch]
+
+define internal i32 @dead_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @live_and_dead_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @dead_callee(i32 %x), !prof !2
+ ret i32 %r
+
+dead:
+ %d = call i32 @dead_callee(i32 %x), !prof !3
+ ret i32 %d
+}
+
+define internal i32 @only_dead_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @only_dead_caller(i32 %x) !prof !1 {
+entry:
+ ret i32 %x
+
+dead:
+ %d = call i32 @only_dead_callee(i32 %x), !prof !3
+ ret i32 %d
+}
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 1}
+!3 = !{!"branch_weights", i32 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 12}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 4}
+!18 = !{!"NumFunctions", i64 4}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-entry-count.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-entry-count.ll
new file mode 100644
index 0000000000000..ecb62c05e9779
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-entry-count.ll
@@ -0,0 +1,88 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=DIAG
+; RUN: opt < %s -passes='function(verify-pgo-flow)' -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=FUNC
+; RUN: not opt < %s -passes=verify-pgo-flow -verify-pgo-flow-fatal \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=FATAL
+;
+; Report only when visible direct-caller weight exceeds entry count.
+; Entry-count vs caller-sum is a module walk; function(verify-pgo-flow)
+; only checks in-function block flow.
+
+; DIAG: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; DIAG-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
+; DIAG-NOT: PGOFlowVerify[EntryCountMismatch] undercount_callee:
+; DIAG: PGOFlowVerify[EntryCountMismatch] bad_callee: entry=1 vs caller-sum=10
+; DIAG: PGOFlowVerify[EntryCountMismatch] real_alias_callee: entry=1 vs caller-sum=10
+; DIAG-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
+; DIAG-NOT: PGOFlowVerify[EntryCountMismatch] undercount_callee:
+
+; FUNC-NOT: PGOFlowVerify[EntryCountMismatch]
+
+; FATAL: PGOFlowVerify[EntryCountMismatch]
+
+define internal i32 @ok_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @ok_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @ok_callee(i32 %x), !prof !2
+ ret i32 %r
+}
+
+define internal i32 @undercount_callee(i32 %x) !prof !3 {
+entry:
+ ret i32 %x
+}
+
+define i32 @undercount_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @undercount_callee(i32 %x), !prof !2
+ ret i32 %r
+}
+
+define internal i32 @bad_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @bad_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @bad_callee(i32 %x), !prof !4
+ ret i32 %r
+}
+
+ at alias_callee = internal alias i32 (i32), ptr @real_alias_callee
+
+define internal i32 @real_alias_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @alias_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @alias_callee(i32 %x), !prof !4
+ ret i32 %r
+}
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 1}
+!3 = !{!"function_entry_count", i64 10}
+!4 = !{!"branch_weights", i32 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 14}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 8}
+!18 = !{!"NumFunctions", i64 8}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-folded-vp-callsite.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-folded-vp-callsite.ll
new file mode 100644
index 0000000000000..1549e0dd08198
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-folded-vp-callsite.ll
@@ -0,0 +1,76 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+;
+; Direct callsites must not use extractProfTotalWeight on leftover VP
+; (operand 2 is the aggregate over every target) or llvm.expect weights.
+; Count-type branch_weights still credit. Unknown sites must not hide a
+; definite overcount from a weighted caller.
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK: PGOFlowVerify[EntryCountMismatch] mixed_callee: entry=1 vs caller-sum=10
+; CHECK-NOT: PGOFlowVerify[EntryCountMismatch] vp_only_callee:
+; CHECK-NOT: PGOFlowVerify[EntryCountMismatch] expected_only_callee:
+
+define internal i32 @mixed_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @folded_vp_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @mixed_callee(i32 %x), !prof !3
+ ret i32 %r
+}
+
+define i32 @expected_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @mixed_callee(i32 %x), !prof !4
+ ret i32 %r
+}
+
+define i32 @weighted_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @mixed_callee(i32 %x), !prof !2
+ ret i32 %r
+}
+
+define internal i32 @vp_only_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @vp_only_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @vp_only_callee(i32 %x), !prof !3
+ ret i32 %r
+}
+
+define internal i32 @expected_only_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @expected_only_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @expected_only_callee(i32 %x), !prof !4
+ ret i32 %r
+}
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 10}
+!3 = !{!"VP", i32 0, i64 100, i64 999, i64 100}
+!4 = !{!"branch_weights", !"expected", i32 50}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 12}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 8}
+!18 = !{!"NumFunctions", i64 8}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unknown-callsite-overcount.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unknown-callsite-overcount.ll
new file mode 100644
index 0000000000000..89de22d19f21f
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unknown-callsite-overcount.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+;
+; Known caller-sum is a lower bound. An unweighted second caller must not
+; hide a definite overcount from a weighted caller of 10 vs entry 1.
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK: PGOFlowVerify[EntryCountMismatch] mixed_callee: entry=1 vs caller-sum=10
+; CHECK-NOT: PGOFlowVerify[EntryCountMismatch] mixed_callee:
+
+define internal i32 @mixed_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @weighted_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @mixed_callee(i32 %x), !prof !2
+ ret i32 %r
+}
+
+define i32 @unweighted_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @mixed_callee(i32 %x)
+ ret i32 %r
+}
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 12}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 3}
+!18 = !{!"NumFunctions", i64 3}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unweighted-callsite.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unweighted-callsite.ll
new file mode 100644
index 0000000000000..954cf601cff02
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unweighted-callsite.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-fatal \
+; RUN: -disable-output 2>&1 | FileCheck %s
+;
+; Two unweighted calls in one block must not each inherit the block SumIn
+; (that would be entry=10 vs caller-sum=20). Treat missing call !prof as
+; unknown and skip the callee's entry-count check.
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK-NOT: PGOFlowVerify[EntryCountMismatch]
+
+define internal i32 @unweighted_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @unweighted_caller(i32 %x) !prof !0 {
+entry:
+ %a = call i32 @unweighted_callee(i32 %x)
+ %b = call i32 @unweighted_callee(i32 %x)
+ ret i32 %b
+}
+
+!0 = !{!"function_entry_count", i64 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 10}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 2}
+!18 = !{!"NumFunctions", i64 2}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
>From f77b7dec07307d352512667a58e0bc7572402159 Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:03:55 +0530
Subject: [PATCH 04/13] [PGOFlowVerify] Add -verify-pgo-flow-funcs allow-list
When the list is non-empty, only named functions are checked. Empty
keeps the current behavior. Callers are still scanned so a listed
callee can sum visible direct-call weights.
---
.../llvm/Transforms/IPO/PGOFlowVerify.h | 1 +
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 32 +++++++++-
.../PGOFlowVerifier/verify-pgo-flow-funcs.ll | 64 +++++++++++++++++++
3 files changed, 94 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-funcs.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
index a31385b4e9296..aea94bf5b14ba 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -55,6 +55,7 @@ class PGOFlowVerifier {
void runAfterPass(const Loop *L);
bool hasInstrProfUseSummary(const Module *M) const;
+ bool shouldVerifyFunction(const Function *F) const;
void computeBlockFrequencies(const Function *F);
void validateBlockFrequencies(const Function *F);
void validateEntryCountAgainstCallerSum(const Function *F);
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
index f5e9e39a57026..1d45fb72184e2 100644
--- a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/IPO/PGOFlowVerify.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
@@ -55,6 +56,10 @@ static cl::opt<bool>
VerifyPGOFlowFatal("verify-pgo-flow-fatal", cl::init(false), cl::Hidden,
cl::desc("Abort after a flow-check finding"));
+static cl::list<std::string> VerifyPGOFlowFuncList(
+ "verify-pgo-flow-funcs", cl::Hidden, cl::CommaSeparated,
+ cl::desc("If non-empty, only verify these functions"));
+
static void printVerifyBanner(StringRef PassName, bool Skipped) {
if (!VerifyPGOFlowPrintDiagnostics)
return;
@@ -167,6 +172,26 @@ void PGOFlowVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
}
}
+bool PGOFlowVerifier::shouldVerifyFunction(const Function *F) const {
+ if (!F || F->isDeclaration())
+ return false;
+ // Non-prevailing copy. The real definition is verified instead.
+ if (F->hasAvailableExternallyLinkage()) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip '" << F->getName()
+ << "' (available_externally)\n");
+ return false;
+ }
+ if (VerifyPGOFlowFuncList.empty())
+ return true;
+ bool Listed = any_of(VerifyPGOFlowFuncList, [&](const std::string &Name) {
+ return !Name.empty() && F->getName() == Name;
+ });
+ if (!Listed)
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip '" << F->getName()
+ << "' (not in -verify-pgo-flow-funcs)\n");
+ return Listed;
+}
+
void PGOFlowVerifier::runAfterPass(const Module *M) {
if (!M)
return;
@@ -181,17 +206,18 @@ void PGOFlowVerifier::runAfterPass(const Module *M) {
if (F.isDeclaration())
continue;
computeBlockFrequencies(&F);
- validateBlockFrequencies(&F);
+ if (shouldVerifyFunction(&F))
+ validateBlockFrequencies(&F);
}
for (const Function &F : *M) {
- if (F.isDeclaration())
+ if (!shouldVerifyFunction(&F))
continue;
validateEntryCountAgainstCallerSum(&F);
}
}
void PGOFlowVerifier::runAfterPass(const Function *F) {
- if (!F || F->isDeclaration() || !F->getParent())
+ if (!F || !F->getParent() || !shouldVerifyFunction(F))
return;
if (!hasInstrProfUseSummary(F->getParent())) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip '" << F->getName()
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-funcs.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-funcs.ll
new file mode 100644
index 0000000000000..b808a334929e3
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-funcs.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-funcs=ok_callee \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=OK
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-funcs=bad_callee \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=BAD
+; RUN: opt < %s -passes=verify-pgo-flow \
+; RUN: -verify-pgo-flow-funcs=ok_callee,bad_callee -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=BOTH
+;
+; Empty -verify-pgo-flow-funcs checks every function. A non-empty list is an
+; allow-list. Callers are still used to sum weights when the callee is listed.
+
+; OK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; OK-NOT: PGOFlowVerify[
+
+; BAD: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; BAD-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
+; BAD: PGOFlowVerify[EntryCountMismatch] bad_callee: entry=1 vs caller-sum=10
+; BAD-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
+
+; BOTH: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; BOTH-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
+; BOTH: PGOFlowVerify[EntryCountMismatch] bad_callee: entry=1 vs caller-sum=10
+; BOTH-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
+
+define internal i32 @ok_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @ok_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @ok_callee(i32 %x), !prof !2
+ ret i32 %r
+}
+
+define internal i32 @bad_callee(i32 %x) !prof !0 {
+entry:
+ ret i32 %x
+}
+
+define i32 @bad_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @bad_callee(i32 %x), !prof !3
+ ret i32 %r
+}
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 1}
+!3 = !{!"branch_weights", i32 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 13}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 1}
+!17 = !{!"NumCounts", i64 4}
+!18 = !{!"NumFunctions", i64 4}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
>From 995e9de977f9edc55d571750a7ef2e3cd05be60e Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:04:10 +0530
Subject: [PATCH 05/13] [IR] Add approxprofile and mark scaled profile counts
Introduce a function attribute for counts that are no longer absolute.
Stamp it from scaleProfData when a weight is clamped. Intersection uses
IntersectOr so either side keeps the attribute, inlining and
MergeFunctions OR it onto the caller and survivor.
---
clang/test/CodeGen/cfi-check-fail-nomerge.c | 239 +++++++++---------
clang/test/CodeGen/cfi-check-fail2-nomerge.c | 124 ++++-----
llvm/docs/LangRef.md | 7 +
llvm/docs/ReleaseNotes.md | 3 +
llvm/include/llvm/Bitcode/LLVMBitCodes.h | 1 +
llvm/include/llvm/IR/Attributes.h | 1 +
llvm/include/llvm/IR/Attributes.td | 10 +
llvm/include/llvm/IR/ProfDataUtils.h | 17 +-
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 2 +
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 2 +
llvm/lib/IR/Attributes.cpp | 24 +-
llvm/lib/IR/ProfDataUtils.cpp | 29 ++-
llvm/lib/Transforms/IPO/MergeFunctions.cpp | 13 +-
llvm/lib/Transforms/Utils/CodeExtractor.cpp | 1 +
.../Transforms/Utils/FunctionComparator.cpp | 9 +-
llvm/test/Bitcode/attributes.ll | 6 +
.../Transforms/CodeExtractor/approxprofile.ll | 27 ++
llvm/test/Transforms/Inline/approxprofile.ll | 92 +++++++
.../Transforms/MergeFunc/approxprofile.ll | 28 ++
.../SimplifyCFG/switch-case-weight-clamp.ll | 44 ++++
llvm/unittests/IR/AttributesTest.cpp | 13 +-
llvm/utils/TableGen/Basic/Attributes.cpp | 4 +
22 files changed, 506 insertions(+), 190 deletions(-)
create mode 100644 llvm/test/Transforms/CodeExtractor/approxprofile.ll
create mode 100644 llvm/test/Transforms/Inline/approxprofile.ll
create mode 100644 llvm/test/Transforms/MergeFunc/approxprofile.ll
create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-case-weight-clamp.ll
diff --git a/clang/test/CodeGen/cfi-check-fail-nomerge.c b/clang/test/CodeGen/cfi-check-fail-nomerge.c
index 05ea8d8327904..2977f69e4301b 100644
--- a/clang/test/CodeGen/cfi-check-fail-nomerge.c
+++ b/clang/test/CodeGen/cfi-check-fail-nomerge.c
@@ -1,5 +1,4 @@
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
-//
// N.B. although the clang driver defaults to merge, clang_cc1 defaults to non-merge.
// (This is similar to -fsanitize-recover, for which the default is also applied
// at the driver level only.)
@@ -36,197 +35,211 @@ void caller(void (*f)(void)) {
+
+
// MERGE-LABEL: define dso_local void @caller(
-// MERGE-SAME: ptr noundef [[F:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !type [[META4:![0-9]+]] !type [[META5:![0-9]+]] !type [[META6:![0-9]+]] {
+// MERGE-SAME: ptr noundef [[F:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] !type [[META10:![0-9]+]] {
// MERGE-NEXT: [[ENTRY:.*:]]
-// MERGE-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.type.test(ptr [[F]], metadata !"_ZTSFvvE"), !nosanitize [[META7:![0-9]+]]
-// MERGE-NEXT: br i1 [[TMP0]], label %[[CFI_CONT:.*]], label %[[CFI_SLOWPATH:.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.type.test(ptr [[F]], metadata !"_ZTSFvvE"), !nosanitize [[META11:![0-9]+]]
+// MERGE-NEXT: br i1 [[TMP0]], label %[[CFI_CONT:.*]], label %[[CFI_SLOWPATH:.*]], !prof [[PROF12:![0-9]+]], !nosanitize [[META11]]
// MERGE: [[CFI_SLOWPATH]]:
-// MERGE-NEXT: tail call void @__cfi_slowpath(i64 9080559750644022485, ptr [[F]]) #[[ATTR5:[0-9]+]], !nosanitize [[META7]]
-// MERGE-NEXT: br label %[[CFI_CONT]], !nosanitize [[META7]]
+// MERGE-NEXT: tail call void @__cfi_slowpath(i64 9080559750644022485, ptr [[F]]) #[[ATTR6:[0-9]+]], !nosanitize [[META11]]
+// MERGE-NEXT: br label %[[CFI_CONT]], !nosanitize [[META11]]
// MERGE: [[CFI_CONT]]:
-// MERGE-NEXT: tail call void [[F]]() #[[ATTR5]]
+// MERGE-NEXT: tail call void [[F]]() #[[ATTR6]]
// MERGE-NEXT: ret void
//
//
// MERGE-LABEL: define weak_odr hidden void @__cfi_check_fail(
-// MERGE-SAME: ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR0]] {
+// MERGE-SAME: ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR2:[0-9]+]] {
// MERGE-NEXT: [[ENTRY:.*:]]
-// MERGE-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP0]], null, !nosanitize [[META7]]
-// MERGE-NEXT: br i1 [[DOTNOT]], label %[[TRAP:.*]], label %[[CONT:.*]], !prof [[PROF9:![0-9]+]], !nosanitize [[META7]]
+// MERGE-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP0]], null, !nosanitize [[META11]]
+// MERGE-NEXT: br i1 [[DOTNOT]], label %[[TRAP:.*]], label %[[CONT:.*]], !prof [[PROF13:![0-9]+]], !nosanitize [[META11]]
// MERGE: [[TRAP]]:
-// MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR6:[0-9]+]], !nosanitize [[META7]]
-// MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7:[0-9]+]], !nosanitize [[META11]]
+// MERGE-NEXT: unreachable, !nosanitize [[META11]]
// MERGE: [[CONT]]:
-// MERGE-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP0]], align 4, !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP3:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP0]], align 4, !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP3:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64, !nosanitize [[META11]]
// MERGE-NEXT: switch i8 [[TMP2]], label %[[CONT8:.*]] [
// MERGE-NEXT: i8 0, label %[[HANDLER_CFI_CHECK_FAIL:.*]]
// MERGE-NEXT: i8 1, label %[[TRAP]]
// MERGE-NEXT: i8 2, label %[[HANDLER_CFI_CHECK_FAIL4:.*]]
// MERGE-NEXT: i8 3, label %[[HANDLER_CFI_CHECK_FAIL6:.*]]
// MERGE-NEXT: i8 4, label %[[TRAP]]
-// MERGE-NEXT: ], !prof [[PROF10:![0-9]+]]
+// MERGE-NEXT: ], !prof [[PROF14:![0-9]+]]
// MERGE: [[HANDLER_CFI_CHECK_FAIL]]:
-// MERGE-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP5]], i64 [[TMP6]], i64 [[TMP4]]) #[[ATTR7:[0-9]+]], !nosanitize [[META7]]
-// MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP5]], i64 [[TMP6]], i64 [[TMP4]]) #[[ATTR8:[0-9]+]], !nosanitize [[META11]]
+// MERGE-NEXT: unreachable, !nosanitize [[META11]]
// MERGE: [[HANDLER_CFI_CHECK_FAIL4]]:
-// MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP8:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP7]], i64 [[TMP8]], i64 [[TMP4]]) #[[ATTR7]], !nosanitize [[META7]]
-// MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP8:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP7]], i64 [[TMP8]], i64 [[TMP4]]) #[[ATTR8]], !nosanitize [[META11]]
+// MERGE-NEXT: unreachable, !nosanitize [[META11]]
// MERGE: [[HANDLER_CFI_CHECK_FAIL6]]:
-// MERGE-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP9]], i64 [[TMP10]], i64 [[TMP4]]) #[[ATTR7]], !nosanitize [[META7]]
-// MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP9]], i64 [[TMP10]], i64 [[TMP4]]) #[[ATTR8]], !nosanitize [[META11]]
+// MERGE-NEXT: unreachable, !nosanitize [[META11]]
// MERGE: [[CONT8]]:
-// MERGE-NEXT: ret void, !nosanitize [[META7]]
+// MERGE-NEXT: ret void, !nosanitize [[META11]]
//
//
// MERGE-LABEL: define weak void @__cfi_check(
-// MERGE-SAME: i64 noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] align 4096 {
+// MERGE-SAME: i64 noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] align 4096 {
// MERGE-NEXT: [[ENTRY:.*:]]
-// MERGE-NEXT: [[DOTNOT_I:%.*]] = icmp eq ptr [[TMP2]], null, !nosanitize [[META7]]
-// MERGE-NEXT: br i1 [[DOTNOT_I]], label %[[TRAP_I:.*]], label %[[CONT_I:.*]], !prof [[PROF9]], !nosanitize [[META7]]
+// MERGE-NEXT: [[DOTNOT_I:%.*]] = icmp eq ptr [[TMP2]], null, !nosanitize [[META11]]
+// MERGE-NEXT: br i1 [[DOTNOT_I]], label %[[TRAP_I:.*]], label %[[CONT_I:.*]], !prof [[PROF13]], !nosanitize [[META11]]
// MERGE: [[TRAP_I]]:
-// MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR6]], !nosanitize [[META7]]
-// MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7]], !nosanitize [[META11]]
+// MERGE-NEXT: unreachable, !nosanitize [[META11]]
// MERGE: [[CONT_I]]:
-// MERGE-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 4, !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i64, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 4, !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i64, !nosanitize [[META11]]
// MERGE-NEXT: switch i8 [[TMP3]], label %[[__CFI_CHECK_FAIL_EXIT:.*]] [
// MERGE-NEXT: i8 0, label %[[HANDLER_CFI_CHECK_FAIL_I:.*]]
// MERGE-NEXT: i8 1, label %[[TRAP_I]]
// MERGE-NEXT: i8 2, label %[[HANDLER_CFI_CHECK_FAIL4_I:.*]]
// MERGE-NEXT: i8 3, label %[[HANDLER_CFI_CHECK_FAIL6_I:.*]]
// MERGE-NEXT: i8 4, label %[[TRAP_I]]
-// MERGE-NEXT: ], !prof [[PROF10]]
+// MERGE-NEXT: ], !prof [[PROF14]]
// MERGE: [[HANDLER_CFI_CHECK_FAIL_I]]:
-// MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP6]], i64 [[TMP7]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META7]]
-// MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP6]], i64 [[TMP7]], i64 [[TMP5]]) #[[ATTR8]], !nosanitize [[META11]]
+// MERGE-NEXT: unreachable, !nosanitize [[META11]]
// MERGE: [[HANDLER_CFI_CHECK_FAIL4_I]]:
-// MERGE-NEXT: [[TMP8:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP8]], i64 [[TMP9]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META7]]
-// MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP8:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP8]], i64 [[TMP9]], i64 [[TMP5]]) #[[ATTR8]], !nosanitize [[META11]]
+// MERGE-NEXT: unreachable, !nosanitize [[META11]]
// MERGE: [[HANDLER_CFI_CHECK_FAIL6_I]]:
-// MERGE-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP10]], i64 [[TMP11]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META7]]
-// MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP10]], i64 [[TMP11]], i64 [[TMP5]]) #[[ATTR8]], !nosanitize [[META11]]
+// MERGE-NEXT: unreachable, !nosanitize [[META11]]
// MERGE: [[__CFI_CHECK_FAIL_EXIT]]:
// MERGE-NEXT: ret void
//
//
// NO-MERGE-LABEL: define dso_local void @caller(
-// NO-MERGE-SAME: ptr noundef [[F:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !type [[META4:![0-9]+]] !type [[META5:![0-9]+]] !type [[META6:![0-9]+]] {
+// NO-MERGE-SAME: ptr noundef [[F:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] !type [[META10:![0-9]+]] {
// NO-MERGE-NEXT: [[ENTRY:.*:]]
-// NO-MERGE-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.type.test(ptr [[F]], metadata !"_ZTSFvvE"), !nosanitize [[META7:![0-9]+]]
-// NO-MERGE-NEXT: br i1 [[TMP0]], label %[[CFI_CONT:.*]], label %[[CFI_SLOWPATH:.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.type.test(ptr [[F]], metadata !"_ZTSFvvE"), !nosanitize [[META11:![0-9]+]]
+// NO-MERGE-NEXT: br i1 [[TMP0]], label %[[CFI_CONT:.*]], label %[[CFI_SLOWPATH:.*]], !prof [[PROF12:![0-9]+]], !nosanitize [[META11]]
// NO-MERGE: [[CFI_SLOWPATH]]:
-// NO-MERGE-NEXT: tail call void @__cfi_slowpath(i64 9080559750644022485, ptr [[F]]) #[[ATTR5:[0-9]+]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: br label %[[CFI_CONT]], !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @__cfi_slowpath(i64 9080559750644022485, ptr [[F]]) #[[ATTR6:[0-9]+]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: br label %[[CFI_CONT]], !nosanitize [[META11]]
// NO-MERGE: [[CFI_CONT]]:
-// NO-MERGE-NEXT: tail call void [[F]]() #[[ATTR5]]
+// NO-MERGE-NEXT: tail call void [[F]]() #[[ATTR6]]
// NO-MERGE-NEXT: ret void
//
//
// NO-MERGE-LABEL: define weak_odr hidden void @__cfi_check_fail(
-// NO-MERGE-SAME: ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR0]] {
+// NO-MERGE-SAME: ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR2:[0-9]+]] {
// NO-MERGE-NEXT: [[ENTRY:.*:]]
-// NO-MERGE-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP0]], null, !nosanitize [[META7]]
-// NO-MERGE-NEXT: br i1 [[DOTNOT]], label %[[TRAP:.*]], label %[[CONT:.*]], !prof [[PROF9:![0-9]+]], !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP0]], null, !nosanitize [[META11]]
+// NO-MERGE-NEXT: br i1 [[DOTNOT]], label %[[TRAP:.*]], label %[[CONT:.*]], !prof [[PROF13:![0-9]+]], !nosanitize [[META11]]
// NO-MERGE: [[TRAP]]:
-// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR6:[0-9]+]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7:[0-9]+]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[CONT]]:
-// NO-MERGE-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP0]], align 4, !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP3:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP0]], align 4, !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP3:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64, !nosanitize [[META11]]
// NO-MERGE-NEXT: switch i8 [[TMP2]], label %[[CONT10:.*]] [
// NO-MERGE-NEXT: i8 0, label %[[HANDLER_CFI_CHECK_FAIL:.*]]
// NO-MERGE-NEXT: i8 1, label %[[TRAP3:.*]]
// NO-MERGE-NEXT: i8 2, label %[[HANDLER_CFI_CHECK_FAIL5:.*]]
// NO-MERGE-NEXT: i8 3, label %[[HANDLER_CFI_CHECK_FAIL7:.*]]
// NO-MERGE-NEXT: i8 4, label %[[TRAP9:.*]]
-// NO-MERGE-NEXT: ], !prof [[PROF10:![0-9]+]]
+// NO-MERGE-NEXT: ], !prof [[PROF14:![0-9]+]]
// NO-MERGE: [[HANDLER_CFI_CHECK_FAIL]]:
-// NO-MERGE-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP5]], i64 [[TMP6]], i64 [[TMP4]]) #[[ATTR7:[0-9]+]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP5]], i64 [[TMP6]], i64 [[TMP4]]) #[[ATTR8:[0-9]+]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[TRAP3]]:
-// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[HANDLER_CFI_CHECK_FAIL5]]:
-// NO-MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP8:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP7]], i64 [[TMP8]], i64 [[TMP4]]) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP8:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP7]], i64 [[TMP8]], i64 [[TMP4]]) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[HANDLER_CFI_CHECK_FAIL7]]:
-// NO-MERGE-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP9]], i64 [[TMP10]], i64 [[TMP4]]) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP9]], i64 [[TMP10]], i64 [[TMP4]]) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[TRAP9]]:
-// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[CONT10]]:
-// NO-MERGE-NEXT: ret void, !nosanitize [[META7]]
+// NO-MERGE-NEXT: ret void, !nosanitize [[META11]]
//
//
// NO-MERGE-LABEL: define weak void @__cfi_check(
-// NO-MERGE-SAME: i64 noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] align 4096 {
+// NO-MERGE-SAME: i64 noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] align 4096 {
// NO-MERGE-NEXT: [[ENTRY:.*:]]
-// NO-MERGE-NEXT: [[DOTNOT_I:%.*]] = icmp eq ptr [[TMP2]], null, !nosanitize [[META7]]
-// NO-MERGE-NEXT: br i1 [[DOTNOT_I]], label %[[TRAP_I:.*]], label %[[CONT_I:.*]], !prof [[PROF9]], !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[DOTNOT_I:%.*]] = icmp eq ptr [[TMP2]], null, !nosanitize [[META11]]
+// NO-MERGE-NEXT: br i1 [[DOTNOT_I]], label %[[TRAP_I:.*]], label %[[CONT_I:.*]], !prof [[PROF13]], !nosanitize [[META11]]
// NO-MERGE: [[TRAP_I]]:
-// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR6]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[CONT_I]]:
-// NO-MERGE-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 4, !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i64, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 4, !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i64, !nosanitize [[META11]]
// NO-MERGE-NEXT: switch i8 [[TMP3]], label %[[__CFI_CHECK_FAIL_EXIT:.*]] [
// NO-MERGE-NEXT: i8 0, label %[[HANDLER_CFI_CHECK_FAIL_I:.*]]
// NO-MERGE-NEXT: i8 1, label %[[TRAP3_I:.*]]
// NO-MERGE-NEXT: i8 2, label %[[HANDLER_CFI_CHECK_FAIL5_I:.*]]
// NO-MERGE-NEXT: i8 3, label %[[HANDLER_CFI_CHECK_FAIL7_I:.*]]
// NO-MERGE-NEXT: i8 4, label %[[TRAP9_I:.*]]
-// NO-MERGE-NEXT: ], !prof [[PROF10]]
+// NO-MERGE-NEXT: ], !prof [[PROF14]]
// NO-MERGE: [[HANDLER_CFI_CHECK_FAIL_I]]:
-// NO-MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP6]], i64 [[TMP7]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP6]], i64 [[TMP7]], i64 [[TMP5]]) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[TRAP3_I]]:
-// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[HANDLER_CFI_CHECK_FAIL5_I]]:
-// NO-MERGE-NEXT: [[TMP8:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP8]], i64 [[TMP9]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP8:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP8]], i64 [[TMP9]], i64 [[TMP5]]) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[HANDLER_CFI_CHECK_FAIL7_I]]:
-// NO-MERGE-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
-// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP10]], i64 [[TMP11]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META11]]
+// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP10]], i64 [[TMP11]], i64 [[TMP5]]) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[TRAP9_I]]:
-// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7]], !nosanitize [[META7]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR8]], !nosanitize [[META11]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META11]]
// NO-MERGE: [[__CFI_CHECK_FAIL_EXIT]]:
// NO-MERGE-NEXT: ret void
-
-// MERGE: [[ATTR5]] = { nounwind }
-// MERGE: [[ATTR6]] = { noreturn nounwind }
-
-// NO-MERGE: [[ATTR6]] = { noreturn nounwind }
-// NO-MERGE: [[ATTR7]] = { nomerge noreturn nounwind }
+//
+//.
+// MERGE: [[META8]] = !{i64 0, !"_ZTSFvPFvvEE"}
+// MERGE: [[META9]] = !{i64 0, !"_ZTSFvPvE.generalized"}
+// MERGE: [[META10]] = !{i64 0, i64 2451761621477796417}
+// MERGE: [[META11]] = !{}
+// MERGE: [[PROF12]] = !{!"branch_weights", i32 1048575, i32 1}
+// MERGE: [[PROF13]] = !{!"branch_weights", i32 1, i32 1048575}
+// MERGE: [[PROF14]] = !{!"branch_weights", i32 -20480, i32 4096, i32 4095, i32 4095, i32 4095, i32 4095}
+//.
+// NO-MERGE: [[META8]] = !{i64 0, !"_ZTSFvPFvvEE"}
+// NO-MERGE: [[META9]] = !{i64 0, !"_ZTSFvPvE.generalized"}
+// NO-MERGE: [[META10]] = !{i64 0, i64 2451761621477796417}
+// NO-MERGE: [[META11]] = !{}
+// NO-MERGE: [[PROF12]] = !{!"branch_weights", i32 1048575, i32 1}
+// NO-MERGE: [[PROF13]] = !{!"branch_weights", i32 1, i32 1048575}
+// NO-MERGE: [[PROF14]] = !{!"branch_weights", i32 -20480, i32 4096, i32 4095, i32 4095, i32 4095, i32 4095}
+//.
diff --git a/clang/test/CodeGen/cfi-check-fail2-nomerge.c b/clang/test/CodeGen/cfi-check-fail2-nomerge.c
index 8fece7e4d60db..d87be8358ec7b 100644
--- a/clang/test/CodeGen/cfi-check-fail2-nomerge.c
+++ b/clang/test/CodeGen/cfi-check-fail2-nomerge.c
@@ -79,62 +79,62 @@ void caller(void (*f)(void)) {
// NO-MERGE-LABEL: define dso_local void @caller(
// NO-MERGE-SAME: ptr nofree noundef readonly captures(none) [[F:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
// NO-MERGE-NEXT: [[ENTRY:.*:]]
-// NO-MERGE-NEXT: tail call void [[F]]() #[[ATTR5:[0-9]+]]
+// NO-MERGE-NEXT: tail call void [[F]]() #[[ATTR6:[0-9]+]]
// NO-MERGE-NEXT: ret void
//
//
// NO-MERGE-LABEL: define weak_odr hidden void @__cfi_check_fail(
-// NO-MERGE-SAME: ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR0]] {
+// NO-MERGE-SAME: ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR1:[0-9]+]] {
// NO-MERGE-NEXT: [[ENTRY:.*:]]
-// NO-MERGE-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP0]], null, !nosanitize [[META6:![0-9]+]]
-// NO-MERGE-NEXT: br i1 [[DOTNOT]], label %[[TRAP:.*]], label %[[CONT:.*]], !prof [[PROF7:![0-9]+]], !nosanitize [[META6]]
+// NO-MERGE-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP0]], null, !nosanitize [[META7:![0-9]+]]
+// NO-MERGE-NEXT: br i1 [[DOTNOT]], label %[[TRAP:.*]], label %[[CONT:.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META7]]
// NO-MERGE: [[TRAP]]:
-// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR6:[0-9]+]], !nosanitize [[META6]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META6]]
+// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7:[0-9]+]], !nosanitize [[META7]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
// NO-MERGE: [[CONT]]:
-// NO-MERGE-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP0]], align 4, !nosanitize [[META6]]
+// NO-MERGE-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP0]], align 4, !nosanitize [[META7]]
// NO-MERGE-NEXT: switch i8 [[TMP2]], label %[[CONT6:.*]] [
// NO-MERGE-NEXT: i8 0, label %[[HANDLER_CFI_CHECK_FAIL:.*]]
// NO-MERGE-NEXT: i8 1, label %[[TRAP]]
// NO-MERGE-NEXT: i8 2, label %[[TRAP]]
// NO-MERGE-NEXT: i8 3, label %[[TRAP]]
// NO-MERGE-NEXT: i8 4, label %[[TRAP]]
-// NO-MERGE-NEXT: ], !prof [[PROF8:![0-9]+]]
+// NO-MERGE-NEXT: ], !prof [[PROF9:![0-9]+]]
// NO-MERGE: [[HANDLER_CFI_CHECK_FAIL]]:
-// NO-MERGE-NEXT: [[TMP3:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META6]]
-// NO-MERGE-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64, !nosanitize [[META6]]
-// NO-MERGE-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META6]]
-// NO-MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META6]]
-// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP5]], i64 [[TMP6]], i64 [[TMP4]]) #[[ATTR7:[0-9]+]], !nosanitize [[META6]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META6]]
+// NO-MERGE-NEXT: [[TMP3:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP5]], i64 [[TMP6]], i64 [[TMP4]]) #[[ATTR8:[0-9]+]], !nosanitize [[META7]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
// NO-MERGE: [[CONT6]]:
-// NO-MERGE-NEXT: ret void, !nosanitize [[META6]]
+// NO-MERGE-NEXT: ret void, !nosanitize [[META7]]
//
//
// NO-MERGE-LABEL: define weak void @__cfi_check(
-// NO-MERGE-SAME: i64 noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] align 4096 {
+// NO-MERGE-SAME: i64 noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] align 4096 {
// NO-MERGE-NEXT: [[ENTRY:.*:]]
-// NO-MERGE-NEXT: [[DOTNOT_I:%.*]] = icmp eq ptr [[TMP2]], null, !nosanitize [[META6]]
-// NO-MERGE-NEXT: br i1 [[DOTNOT_I]], label %[[TRAP_I:.*]], label %[[CONT_I:.*]], !prof [[PROF7]], !nosanitize [[META6]]
+// NO-MERGE-NEXT: [[DOTNOT_I:%.*]] = icmp eq ptr [[TMP2]], null, !nosanitize [[META7]]
+// NO-MERGE-NEXT: br i1 [[DOTNOT_I]], label %[[TRAP_I:.*]], label %[[CONT_I:.*]], !prof [[PROF8]], !nosanitize [[META7]]
// NO-MERGE: [[TRAP_I]]:
-// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR6]], !nosanitize [[META6]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META6]]
+// NO-MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7]], !nosanitize [[META7]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
// NO-MERGE: [[CONT_I]]:
-// NO-MERGE-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 4, !nosanitize [[META6]]
+// NO-MERGE-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 4, !nosanitize [[META7]]
// NO-MERGE-NEXT: switch i8 [[TMP3]], label %[[__CFI_CHECK_FAIL_EXIT:.*]] [
// NO-MERGE-NEXT: i8 0, label %[[HANDLER_CFI_CHECK_FAIL_I:.*]]
// NO-MERGE-NEXT: i8 1, label %[[TRAP_I]]
// NO-MERGE-NEXT: i8 2, label %[[TRAP_I]]
// NO-MERGE-NEXT: i8 3, label %[[TRAP_I]]
// NO-MERGE-NEXT: i8 4, label %[[TRAP_I]]
-// NO-MERGE-NEXT: ], !prof [[PROF8]]
+// NO-MERGE-NEXT: ], !prof [[PROF9]]
// NO-MERGE: [[HANDLER_CFI_CHECK_FAIL_I]]:
-// NO-MERGE-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META6]]
-// NO-MERGE-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i64, !nosanitize [[META6]]
-// NO-MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META6]]
-// NO-MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META6]]
-// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP6]], i64 [[TMP7]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META6]]
-// NO-MERGE-NEXT: unreachable, !nosanitize [[META6]]
+// NO-MERGE-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i64, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META7]]
+// NO-MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
+// NO-MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP6]], i64 [[TMP7]], i64 [[TMP5]]) #[[ATTR8]], !nosanitize [[META7]]
+// NO-MERGE-NEXT: unreachable, !nosanitize [[META7]]
// NO-MERGE: [[__CFI_CHECK_FAIL_EXIT]]:
// NO-MERGE-NEXT: ret void
//
@@ -142,71 +142,71 @@ void caller(void (*f)(void)) {
// MERGE-LABEL: define dso_local void @caller(
// MERGE-SAME: ptr nofree noundef readonly captures(none) [[F:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
// MERGE-NEXT: [[ENTRY:.*:]]
-// MERGE-NEXT: tail call void [[F]]() #[[ATTR5:[0-9]+]]
+// MERGE-NEXT: tail call void [[F]]() #[[ATTR6:[0-9]+]]
// MERGE-NEXT: ret void
//
//
// MERGE-LABEL: define weak_odr hidden void @__cfi_check_fail(
-// MERGE-SAME: ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR0]] {
+// MERGE-SAME: ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR1:[0-9]+]] {
// MERGE-NEXT: [[ENTRY:.*:]]
-// MERGE-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP0]], null, !nosanitize [[META6:![0-9]+]]
-// MERGE-NEXT: br i1 [[DOTNOT]], label %[[TRAP:.*]], label %[[CONT:.*]], !prof [[PROF7:![0-9]+]], !nosanitize [[META6]]
+// MERGE-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP0]], null, !nosanitize [[META7:![0-9]+]]
+// MERGE-NEXT: br i1 [[DOTNOT]], label %[[TRAP:.*]], label %[[CONT:.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META7]]
// MERGE: [[TRAP]]:
-// MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR6:[0-9]+]], !nosanitize [[META6]]
-// MERGE-NEXT: unreachable, !nosanitize [[META6]]
+// MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7:[0-9]+]], !nosanitize [[META7]]
+// MERGE-NEXT: unreachable, !nosanitize [[META7]]
// MERGE: [[CONT]]:
-// MERGE-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP0]], align 4, !nosanitize [[META6]]
+// MERGE-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP0]], align 4, !nosanitize [[META7]]
// MERGE-NEXT: switch i8 [[TMP2]], label %[[CONT6:.*]] [
// MERGE-NEXT: i8 0, label %[[HANDLER_CFI_CHECK_FAIL:.*]]
// MERGE-NEXT: i8 1, label %[[TRAP]]
// MERGE-NEXT: i8 2, label %[[TRAP]]
// MERGE-NEXT: i8 3, label %[[TRAP]]
// MERGE-NEXT: i8 4, label %[[TRAP]]
-// MERGE-NEXT: ], !prof [[PROF8:![0-9]+]]
+// MERGE-NEXT: ], !prof [[PROF9:![0-9]+]]
// MERGE: [[HANDLER_CFI_CHECK_FAIL]]:
-// MERGE-NEXT: [[TMP3:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META6]]
-// MERGE-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64, !nosanitize [[META6]]
-// MERGE-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META6]]
-// MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META6]]
-// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP5]], i64 [[TMP6]], i64 [[TMP4]]) #[[ATTR6]], !nosanitize [[META6]]
-// MERGE-NEXT: unreachable, !nosanitize [[META6]]
+// MERGE-NEXT: [[TMP3:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP4:%.*]] = zext i1 [[TMP3]] to i64, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP0]] to i64, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
+// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP5]], i64 [[TMP6]], i64 [[TMP4]]) #[[ATTR7]], !nosanitize [[META7]]
+// MERGE-NEXT: unreachable, !nosanitize [[META7]]
// MERGE: [[CONT6]]:
-// MERGE-NEXT: ret void, !nosanitize [[META6]]
+// MERGE-NEXT: ret void, !nosanitize [[META7]]
//
//
// MERGE-LABEL: define weak void @__cfi_check(
-// MERGE-SAME: i64 noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] align 4096 {
+// MERGE-SAME: i64 noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] align 4096 {
// MERGE-NEXT: [[ENTRY:.*:]]
-// MERGE-NEXT: [[DOTNOT_I:%.*]] = icmp eq ptr [[TMP2]], null, !nosanitize [[META6]]
-// MERGE-NEXT: br i1 [[DOTNOT_I]], label %[[TRAP_I:.*]], label %[[CONT_I:.*]], !prof [[PROF7]], !nosanitize [[META6]]
+// MERGE-NEXT: [[DOTNOT_I:%.*]] = icmp eq ptr [[TMP2]], null, !nosanitize [[META7]]
+// MERGE-NEXT: br i1 [[DOTNOT_I]], label %[[TRAP_I:.*]], label %[[CONT_I:.*]], !prof [[PROF8]], !nosanitize [[META7]]
// MERGE: [[TRAP_I]]:
-// MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR6]], !nosanitize [[META6]]
-// MERGE-NEXT: unreachable, !nosanitize [[META6]]
+// MERGE-NEXT: tail call void @llvm.ubsantrap(i8 2) #[[ATTR7]], !nosanitize [[META7]]
+// MERGE-NEXT: unreachable, !nosanitize [[META7]]
// MERGE: [[CONT_I]]:
-// MERGE-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 4, !nosanitize [[META6]]
+// MERGE-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 4, !nosanitize [[META7]]
// MERGE-NEXT: switch i8 [[TMP3]], label %[[__CFI_CHECK_FAIL_EXIT:.*]] [
// MERGE-NEXT: i8 0, label %[[HANDLER_CFI_CHECK_FAIL_I:.*]]
// MERGE-NEXT: i8 1, label %[[TRAP_I]]
// MERGE-NEXT: i8 2, label %[[TRAP_I]]
// MERGE-NEXT: i8 3, label %[[TRAP_I]]
// MERGE-NEXT: i8 4, label %[[TRAP_I]]
-// MERGE-NEXT: ], !prof [[PROF8]]
+// MERGE-NEXT: ], !prof [[PROF9]]
// MERGE: [[HANDLER_CFI_CHECK_FAIL_I]]:
-// MERGE-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META6]]
-// MERGE-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i64, !nosanitize [[META6]]
-// MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META6]]
-// MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META6]]
-// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP6]], i64 [[TMP7]], i64 [[TMP5]]) #[[ATTR6]], !nosanitize [[META6]]
-// MERGE-NEXT: unreachable, !nosanitize [[META6]]
+// MERGE-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.type.test(ptr [[TMP1]], metadata !"all-vtables"), !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i64, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP2]] to i64, !nosanitize [[META7]]
+// MERGE-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64, !nosanitize [[META7]]
+// MERGE-NEXT: tail call void @__ubsan_handle_cfi_check_fail_abort(i64 [[TMP6]], i64 [[TMP7]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META7]]
+// MERGE-NEXT: unreachable, !nosanitize [[META7]]
// MERGE: [[__CFI_CHECK_FAIL_EXIT]]:
// MERGE-NEXT: ret void
//
//.
-// NO-MERGE: [[META6]] = !{}
-// NO-MERGE: [[PROF7]] = !{!"branch_weights", i32 1, i32 1048575}
-// NO-MERGE: [[PROF8]] = !{!"branch_weights", i32 -20480, i32 4096, i32 4095, i32 4095, i32 4095, i32 4095}
+// NO-MERGE: [[META7]] = !{}
+// NO-MERGE: [[PROF8]] = !{!"branch_weights", i32 1, i32 1048575}
+// NO-MERGE: [[PROF9]] = !{!"branch_weights", i32 -20480, i32 4096, i32 4095, i32 4095, i32 4095, i32 4095}
//.
-// MERGE: [[META6]] = !{}
-// MERGE: [[PROF7]] = !{!"branch_weights", i32 1, i32 1048575}
-// MERGE: [[PROF8]] = !{!"branch_weights", i32 -20480, i32 4096, i32 4095, i32 4095, i32 4095, i32 4095}
+// MERGE: [[META7]] = !{}
+// MERGE: [[PROF8]] = !{!"branch_weights", i32 1, i32 1048575}
+// MERGE: [[PROF9]] = !{!"branch_weights", i32 -20480, i32 4096, i32 4095, i32 4095, i32 4095, i32 4095}
//.
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 14caff88243c1..77465c1d410d0 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -2492,6 +2492,13 @@ define void @f() "no-sse" { ... }
attribute does not restrict inlining, so instrumented instructions could end
up in this function.
+`approxprofile`
+: Profile counts on this function are approximate. Ratios between counts may
+ still be meaningful, but absolute magnitudes are not. The attribute is
+ function-wide. Attribute intersection keeps it if either side has it.
+ Inlining ORs it onto the caller. MergeFunctions ignores it for
+ equivalence and ORs it onto the survivor. It is not immutable in IR.
+
`noredzone`
: This attribute indicates that the code generator should not use a
red zone, even if the target-specific ABI normally permits it.
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 4718d47bf8c10..983f372a971ab 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -79,6 +79,9 @@ Makes programs 10x faster by doing Special New Thing.
freeing the underlying object (as opposed to only frees through that specific
pointer). Renamed `!nofree` metadata to `!nofreeobj`, as it has the same
semantics.
+* Added the `approxprofile` function attribute to mark profile counts whose
+ absolute magnitudes are no longer trustworthy (ratios may still be). It is
+ sticky: intersection and inlining keep it if either side has it.
* The following VP intrinsics have been removed:
* `llvm.vp.select.*`
* `llvm.vp.add.*`
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 4099b57f5482e..7eb0fee4899a3 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -831,6 +831,7 @@ enum AttributeKindCodes {
ATTR_KIND_FLATTEN = 108,
ATTR_KIND_NOIPA = 109,
ATTR_KIND_NOFREEOBJ = 110,
+ ATTR_KIND_APPROXPROFILE = 111,
};
enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index f8185c76dfb5c..caa8d780bee12 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -160,6 +160,7 @@ class Attribute {
LLVM_ABI static bool intersectMustPreserve(AttrKind Kind);
LLVM_ABI static bool intersectWithAnd(AttrKind Kind);
LLVM_ABI static bool intersectWithMin(AttrKind Kind);
+ LLVM_ABI static bool intersectWithOr(AttrKind Kind);
LLVM_ABI static bool intersectWithCustom(AttrKind Kind);
private:
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index ea523cad69cd5..8701903c7f681 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -42,6 +42,11 @@ def IntersectAnd : AttrProperty;
/// Only valid for Int attrs.
def IntersectMin : AttrProperty;
+/// When intersecting take the OR of the two attrs.
+/// Keep the attribute if either side has it.
+/// Only valid for Enum attrs.
+def IntersectOr : AttrProperty;
+
/// When intersecting rely on some specially defined code.
def IntersectCustom : AttrProperty;
@@ -121,6 +126,10 @@ def Convergent : EnumAttr<"convergent", IntersectPreserve, [FnAttr]>;
/// Marks function as being in a hot path and frequently called.
def Hot: EnumAttr<"hot", IntersectAnd, [FnAttr]>;
+/// Profile counts are approximate (ratios preserved, not absolute magnitudes).
+/// Sticky taint: intersection and inlining keep it if either side has it.
+def ApproxProfile : EnumAttr<"approxprofile", IntersectOr, [FnAttr]>;
+
/// Pointer is known to be dereferenceable.
def Dereferenceable : IntAttr<"dereferenceable", IntersectMin, [ParamAttr, RetAttr]>;
@@ -492,6 +501,7 @@ def : MergeRule<"setOR<NoImplicitFloatAttr>">;
def : MergeRule<"setOR<NoJumpTablesAttr>">;
def : MergeRule<"setOR<ProfileSampleAccurateAttr>">;
def : MergeRule<"setOR<SpeculativeLoadHardeningAttr>">;
+def : MergeRule<"setOR<ApproxProfileAttr>">;
def : MergeRule<"adjustCallerSSPLevel">;
def : MergeRule<"adjustCallerStackProbes">;
def : MergeRule<"adjustCallerStackProbeSize">;
diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h
index 2d77ef33e7056..51926e2862d2a 100644
--- a/llvm/include/llvm/IR/ProfDataUtils.h
+++ b/llvm/include/llvm/IR/ProfDataUtils.h
@@ -24,6 +24,9 @@
#include <type_traits>
namespace llvm {
+class Function;
+class Instruction;
+class MDNode;
struct MDProfLabels {
LLVM_ABI static const char *BranchWeights;
LLVM_ABI static const char *ValueProfile;
@@ -145,6 +148,13 @@ LLVM_ABI bool extractProfTotalWeight(const MDNode *ProfileData,
LLVM_ABI bool extractProfTotalWeight(const Instruction &I,
uint64_t &TotalWeights);
+/// Mark that profile counts attached to \p F are approximate, not absolute.
+/// The attribute is function-wide and sticky (once set, it remains on \p F).
+LLVM_ABI void markApproximateProfileCounts(Function &F);
+
+/// Return true when \p F carries the approxprofile function attribute.
+LLVM_ABI bool hasApproximateProfileCounts(const Function &F);
+
/// Create a new `branch_weights` metadata node and add or overwrite
/// a `prof` metadata reference to instruction `I`.
/// \param I the Instruction to set branch weights on.
@@ -156,8 +166,8 @@ LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
/// Push the weights right to fit in uint32_t.
LLVM_ABI SmallVector<uint32_t> fitWeights(ArrayRef<uint64_t> Weights);
-/// Variant of `setBranchWeights` where the `Weights` will be fit first to
-/// uint32_t by shifting right.
+/// Like \c setBranchWeights after fitting to uint32_t. A shift of non-expect
+/// weights marks the function \c approxprofile.
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef<uint64_t> Weights,
bool IsExpected,
bool ElideAllZero = false);
@@ -222,7 +232,8 @@ LLVM_ABI void setExplicitlyUnknownFunctionEntryCount(Function &F,
LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD);
LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I);
-/// Scaling the profile data attached to 'I' using the ratio of S/T.
+/// Scale the profile data attached to \p I by S/T. Counts that do not fit
+/// in uint32_t are saturated and mark the function \c approxprofile.
LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T);
// Helper to apply a metadata setting function to an Instruction* if profiling
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index d75e85b0b0f9e..35a67bb95da0f 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2310,6 +2310,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
return Attribute::MustProgress;
case bitc::ATTR_KIND_HOT:
return Attribute::Hot;
+ case bitc::ATTR_KIND_APPROXPROFILE:
+ return Attribute::ApproxProfile;
case bitc::ATTR_KIND_PRESPLIT_COROUTINE:
return Attribute::PresplitCoroutine;
case bitc::ATTR_KIND_WRITABLE:
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index e33b6e0050318..47f009eb82abf 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -832,6 +832,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
return bitc::ATTR_KIND_FLATTEN;
case Attribute::Hot:
return bitc::ATTR_KIND_HOT;
+ case Attribute::ApproxProfile:
+ return bitc::ATTR_KIND_APPROXPROFILE;
case Attribute::ElementType:
return bitc::ATTR_KIND_ELEMENTTYPE;
case Attribute::HybridPatchable:
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 727a7bcb6a144..ec144d89fb63d 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -810,7 +810,8 @@ enum AttributeProperty {
IntersectAnd = (1 << 3),
IntersectMin = (2 << 3),
IntersectCustom = (3 << 3),
- IntersectPropertyMask = (3 << 3),
+ IntersectOr = (4 << 3),
+ IntersectPropertyMask = (7 << 3),
};
#define GET_ATTR_PROP_TABLE
@@ -844,7 +845,8 @@ static bool hasIntersectProperty(Attribute::AttrKind Kind,
assert((Prop == AttributeProperty::IntersectPreserve ||
Prop == AttributeProperty::IntersectAnd ||
Prop == AttributeProperty::IntersectMin ||
- Prop == AttributeProperty::IntersectCustom) &&
+ Prop == AttributeProperty::IntersectCustom ||
+ Prop == AttributeProperty::IntersectOr) &&
"Unknown intersect property");
return (getAttributeProperties(Kind) &
AttributeProperty::IntersectPropertyMask) == Prop;
@@ -859,6 +861,9 @@ bool Attribute::intersectWithAnd(AttrKind Kind) {
bool Attribute::intersectWithMin(AttrKind Kind) {
return hasIntersectProperty(Kind, AttributeProperty::IntersectMin);
}
+bool Attribute::intersectWithOr(AttrKind Kind) {
+ return hasIntersectProperty(Kind, AttributeProperty::IntersectOr);
+}
bool Attribute::intersectWithCustom(AttrKind Kind) {
return hasIntersectProperty(Kind, AttributeProperty::IntersectCustom);
}
@@ -1091,8 +1096,12 @@ AttributeSet::intersectWith(LLVMContext &C, AttributeSet Other) const {
Attribute::AttrKind Kind = Attr0.getKindAsEnum();
// If we don't have both attributes, then fail if the attribute is
- // must-preserve or drop it otherwise.
+ // must-preserve, keep it if it is sticky-OR, or drop it otherwise.
if (!Attr1.isValid()) {
+ if (Attribute::intersectWithOr(Kind)) {
+ Intersected.addAttribute(Kind);
+ continue;
+ }
if (Attribute::intersectMustPreserve(Kind))
return std::nullopt;
continue;
@@ -1118,6 +1127,15 @@ AttributeSet::intersectWith(LLVMContext &C, AttributeSet Other) const {
Intersected.addRawIntAttr(Kind, NewVal);
continue;
}
+
+ // Attribute we can intersect with "or": keep if either side has it.
+ if (Attribute::intersectWithOr(Kind)) {
+ assert(Attribute::isEnumAttrKind(Kind) &&
+ "Invalid attr type of intersectOr");
+ Intersected.addAttribute(Kind);
+ continue;
+ }
+
// Attribute we can intersect but need a custom rule for.
if (Attribute::intersectWithCustom(Kind)) {
switch (Kind) {
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index 34d46cb062bc3..35548a3be89ac 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -15,6 +15,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
@@ -29,6 +30,19 @@ namespace llvm {
extern cl::opt<bool> ProfcheckDisableMetadataFixes;
}
+void llvm::markApproximateProfileCounts(Function &F) {
+ F.addFnAttr(Attribute::ApproxProfile);
+}
+
+bool llvm::hasApproximateProfileCounts(const Function &F) {
+ return F.hasFnAttribute(Attribute::ApproxProfile);
+}
+
+static void markApproximateProfileCountsIfNeeded(Instruction &I) {
+ if (Function *F = I.getFunction())
+ markApproximateProfileCounts(*F);
+}
+
// MD_prof nodes have the following layout
//
// In general:
@@ -335,7 +349,10 @@ void llvm::setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
void llvm::setFittedBranchWeights(Instruction &I, ArrayRef<uint64_t> Weights,
bool IsExpected, bool ElideAllZero) {
+ uint64_t Max = Weights.empty() ? 0 : *llvm::max_element(Weights);
setBranchWeights(I, fitWeights(Weights), IsExpected, ElideAllZero);
+ if (Max > UINT_MAX && !IsExpected)
+ markApproximateProfileCountsIfNeeded(I);
}
SmallVector<uint32_t>
@@ -381,8 +398,11 @@ void llvm::scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
->getValue()
.getZExtValue());
Val *= APS;
+ APInt Scaled = Val.udiv(APT);
+ if (!Scaled.isIntN(32))
+ markApproximateProfileCountsIfNeeded(I);
Vals.push_back(MDB.createConstant(ConstantInt::get(
- Type::getInt32Ty(C), Val.udiv(APT).getLimitedValue(UINT32_MAX))));
+ Type::getInt32Ty(C), Scaled.getLimitedValue(UINT32_MAX))));
} else if (ProfDataName->getString() == MDProfLabels::ValueProfile)
for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx += 2) {
// The first value is the key of the value profile, which will not change.
@@ -399,8 +419,11 @@ void llvm::scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
// Using APInt::div may be expensive, but most cases should fit 64 bits.
APInt Val(128, Count);
Val *= APS;
- Vals.push_back(MDB.createConstant(ConstantInt::get(
- Type::getInt64Ty(C), Val.udiv(APT).getLimitedValue())));
+ APInt Scaled = Val.udiv(APT);
+ if (!Scaled.isIntN(64))
+ markApproximateProfileCountsIfNeeded(I);
+ Vals.push_back(MDB.createConstant(
+ ConstantInt::get(Type::getInt64Ty(C), Scaled.getLimitedValue())));
}
I.setMetadata(LLVMContext::MD_prof, MDNode::get(C, Vals));
}
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index 0101b469b215e..203f1ed098ee4 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -951,6 +951,11 @@ static bool isODR(const Function *F) {
return F->hasWeakODRLinkage() || F->hasLinkOnceODRLinkage();
}
+static void propagateApproximateProfile(Function *F, bool GWasApprox) {
+ if (GWasApprox || hasApproximateProfileCounts(*F))
+ markApproximateProfileCounts(*F);
+}
+
static uint64_t getBlockCountForMerging(const BlockFrequencyInfo &BFI,
const BasicBlock *BB) {
if (auto Count = BFI.getBlockProfileCount(BB, /*AllowSynthetic=*/true))
@@ -1120,7 +1125,7 @@ void MergeFunctions::mergeInstrAnnotations(Function *Dst, Function *Src) {
// Merge two equivalent functions. Upon completion, Function G is deleted.
void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
-
+ const bool GWasApprox = hasApproximateProfileCounts(*G);
std::optional<uint64_t> FEntryCount = F->getEntryCount();
// Create a new thunk that both F and G can call, if F cannot call G directly.
@@ -1177,6 +1182,7 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
else
F->setAlignment(std::nullopt);
F->setLinkage(GlobalValue::PrivateLinkage);
+ propagateApproximateProfile(F, GWasApprox);
++NumDoubleWeak;
++NumFunctionsMerged;
} else {
@@ -1206,13 +1212,16 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
if (G->isDiscardableIfUnused() && G->use_empty() && !MergeFunctionsPDI) {
mergeInstrAnnotations(F, G);
mergeEntryCountsAndImportsInto(*F, *G);
+ propagateApproximateProfile(F, GWasApprox);
G->eraseFromParent();
++NumFunctionsMerged;
return;
}
- if (writeThunkOrAliasIfNeeded(F, G, /*MergeAnnotations=*/true))
+ if (writeThunkOrAliasIfNeeded(F, G, /*MergeAnnotations=*/true)) {
+ propagateApproximateProfile(F, GWasApprox);
++NumFunctionsMerged;
+ }
}
}
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 4f224d0a18e48..124e0d91a4d00 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -979,6 +979,7 @@ Function *CodeExtractor::constructFunctionDeclaration(
case Attribute::Flatten:
case Attribute::FnRetThunkExtern:
case Attribute::Hot:
+ case Attribute::ApproxProfile:
case Attribute::HybridPatchable:
case Attribute::NoRecurse:
case Attribute::InlineHint:
diff --git a/llvm/lib/Transforms/Utils/FunctionComparator.cpp b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
index 9770ae4e39f66..de17e52a24457 100644
--- a/llvm/lib/Transforms/Utils/FunctionComparator.cpp
+++ b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
@@ -119,8 +119,13 @@ int FunctionComparator::cmpMem(StringRef L, StringRef R) const {
return std::clamp(L.compare(R), -1, 1);
}
-int FunctionComparator::cmpAttrs(const AttributeList L,
- const AttributeList R) const {
+int FunctionComparator::cmpAttrs(const AttributeList LIn,
+ const AttributeList RIn) const {
+ LLVMContext &Ctx = FnL->getContext();
+ // approxprofile is a sticky taint, not part of functional equivalence.
+ AttributeList L = LIn.removeFnAttribute(Ctx, Attribute::ApproxProfile);
+ AttributeList R = RIn.removeFnAttribute(Ctx, Attribute::ApproxProfile);
+
if (int Res = cmpNumbers(L.getNumAttrSets(), R.getNumAttrSets()))
return Res;
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index ae5e806a59210..3f7c9b4d4b8ce 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -607,6 +607,11 @@ define nofreeobj ptr @nofreeobj(ptr nofreeobj %p) {
ret ptr %p
}
+; CHECK: define void @f_approxprofile() [[APPROXPROFILE:#[0-9]+]]
+define void @f_approxprofile() approxprofile {
+ ret void
+}
+
; CHECK: attributes #0 = { noreturn }
; CHECK: attributes #1 = { nounwind }
; CHECK: attributes #2 = { memory(none) }
@@ -671,4 +676,5 @@ define nofreeobj ptr @nofreeobj(ptr nofreeobj %p) {
; CHECK: attributes [[OPTDEBUG]] = { optdebug }
; CHECK: attributes [[NODIVERGENCESOURCE]] = { nodivergencesource }
; CHECK: attributes [[NOIPA]] = { noipa }
+; CHECK: attributes [[APPROXPROFILE]] = { approxprofile }
; CHECK: attributes #[[NOBUILTIN]] = { nobuiltin }
diff --git a/llvm/test/Transforms/CodeExtractor/approxprofile.ll b/llvm/test/Transforms/CodeExtractor/approxprofile.ll
new file mode 100644
index 0000000000000..42bbaeb9103fb
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/approxprofile.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -passes=partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck %s
+;
+; CodeExtractor copies approxprofile onto the extracted function.
+
+define i32 @inlinedFunc(i1 %cond) approxprofile !prof !1 {
+entry:
+ br i1 %cond, label %if.then, label %return, !prof !2
+if.then:
+ br i1 %cond, label %if.then, label %return, !prof !3
+return:
+ ret i32 0
+}
+
+define internal i32 @dummyCaller(i1 %cond) !prof !1 {
+entry:
+ %val = call i32 @inlinedFunc(i1 %cond)
+ ret i32 %val
+}
+
+; CHECK: define {{.*}} @inlinedFunc.1.if.then({{.*}}) #[[A:[0-9]+]]
+; CHECK: attributes #[[A]] = { {{.*}}approxprofile{{.*}} }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"MaxFunctionCount", i32 1000}
+!1 = !{!"function_entry_count", i64 1000}
+!2 = !{!"branch_weights", i32 250, i32 750}
+!3 = !{!"branch_weights", i32 125, i32 125}
diff --git a/llvm/test/Transforms/Inline/approxprofile.ll b/llvm/test/Transforms/Inline/approxprofile.ll
new file mode 100644
index 0000000000000..9e1ccd88e4abd
--- /dev/null
+++ b/llvm/test/Transforms/Inline/approxprofile.ll
@@ -0,0 +1,92 @@
+; RUN: opt -passes='require<profile-summary>,inline' -S < %s | FileCheck %s
+;
+; When scaleProfData clamps a weight, the function that holds the scaled
+; metadata is marked approxprofile.
+
+declare void @side_effect()
+
+define internal void @leaf_in_range() noinline {
+entry:
+ call void @side_effect()
+ ret void
+}
+
+define internal void @mid_in_range() !prof !10 {
+entry:
+ call void @leaf_in_range(), !prof !11
+ ret void
+}
+
+; CHECK-LABEL: define void @caller_in_range(){{[[:space:]]+}}!prof
+; CHECK-NOT: #{{[0-9]+}}
+; CHECK: call void @leaf_in_range(), !prof [[INRANGE:![0-9]+]]
+define void @caller_in_range() !prof !12 {
+entry:
+ call void @mid_in_range(), !prof !13
+ ret void
+}
+
+define internal void @leaf_overflow() noinline {
+entry:
+ call void @side_effect()
+ ret void
+}
+
+define internal void @mid_overflow() !prof !20 {
+entry:
+ call void @leaf_overflow(), !prof !21
+ ret void
+}
+
+; CHECK-LABEL: define void @caller_overflow()
+; CHECK-SAME: #[[APPROX:[0-9]+]]
+; CHECK: call void @leaf_overflow(), !prof [[CLAMPED:![0-9]+]]
+define void @caller_overflow() !prof !22 {
+entry:
+ call void @mid_overflow(), !prof !23
+ ret void
+}
+
+define internal void @approx_leaf() alwaysinline approxprofile {
+entry:
+ call void @side_effect()
+ ret void
+}
+
+; CHECK-LABEL: define void @exact_caller()
+; CHECK-SAME: #[[APPROX]]
+define void @exact_caller() {
+entry:
+ call void @approx_leaf()
+ ret void
+}
+
+; CHECK: attributes #[[APPROX]] = { approxprofile }
+; CHECK: [[INRANGE]] = !{!"branch_weights", i32 5}
+; CHECK: [[CLAMPED]] = !{!"branch_weights", i32 -1}
+
+!llvm.module.flags = !{!30}
+!30 = !{i32 1, !"ProfileSummary", !31}
+!31 = !{!32, !33, !34, !35, !36, !37, !38, !39}
+!32 = !{!"ProfileFormat", !"InstrProf"}
+!33 = !{!"TotalCount", i64 400}
+!34 = !{!"MaxCount", i64 100}
+!35 = !{!"MaxInternalCount", i64 100}
+!36 = !{!"MaxFunctionCount", i64 100}
+!37 = !{!"NumCounts", i64 6}
+!38 = !{!"NumFunctions", i64 6}
+!39 = !{!"DetailedSummary", !40}
+!40 = !{!41, !42, !43}
+!41 = !{i32 10000, i64 100, i32 1}
+!42 = !{i32 999000, i64 100, i32 2}
+!43 = !{i32 999999, i64 100, i32 3}
+
+!10 = !{!"function_entry_count", i64 100}
+!11 = !{!"branch_weights", i32 5}
+!12 = !{!"function_entry_count", i64 100}
+!13 = !{!"branch_weights", i64 100}
+
+!20 = !{!"function_entry_count", i64 100}
+!21 = !{!"branch_weights", i64 8000000000}
+!22 = !{!"function_entry_count", i64 100}
+!23 = !{!"branch_weights", i64 100}
diff --git a/llvm/test/Transforms/MergeFunc/approxprofile.ll b/llvm/test/Transforms/MergeFunc/approxprofile.ll
new file mode 100644
index 0000000000000..ed8de95682efe
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/approxprofile.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S -passes=mergefunc < %s | FileCheck %s
+;
+; Mixed approxprofile / exact pairs still merge. The lexicographically
+; smaller name is the unmarked survivor, MergeFunctions ORs approxprofile
+; onto it from @b_approx.
+
+define i32 @a_exact(i32 %x) unnamed_addr {
+entry:
+ %a = add i32 %x, 1
+ %b = add i32 %a, 1
+ %c = add i32 %b, 1
+ %d = add i32 %c, 1
+ ret i32 %d
+}
+
+define i32 @b_approx(i32 %x) unnamed_addr approxprofile {
+entry:
+ %a = add i32 %x, 1
+ %b = add i32 %a, 1
+ %c = add i32 %b, 1
+ %d = add i32 %c, 1
+ ret i32 %d
+}
+
+; CHECK: define i32 @a_exact(i32 %x) unnamed_addr #[[A:[0-9]+]] {
+; CHECK: define i32 @b_approx(i32 %{{.*}}) unnamed_addr #[[A]] {
+; CHECK-NEXT: tail call i32 @a_exact
+; CHECK: attributes #[[A]] = { approxprofile }
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-case-weight-clamp.ll b/llvm/test/Transforms/SimplifyCFG/switch-case-weight-clamp.ll
new file mode 100644
index 0000000000000..3d12289035d0c
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-case-weight-clamp.ll
@@ -0,0 +1,44 @@
+; RUN: split-file %s %t
+; RUN: opt -S -passes=simplifycfg %t/count.ll | FileCheck %s --check-prefix=COUNT
+; RUN: opt -S -passes=simplifycfg %t/expected.ll | FileCheck %s --check-prefix=EXPECTED
+;
+; Overflowing the i32 weight sum marks approxprofile. llvm.expect does not.
+
+; COUNT: Function Attrs: approxprofile
+; COUNT-LABEL: define i32 @count(
+; COUNT: attributes #[[ATTR:[0-9]+]] = { approxprofile }
+
+; EXPECTED-NOT: Function Attrs: approxprofile
+; EXPECTED-NOT: approxprofile
+; EXPECTED-LABEL: define i32 @expected(i32 %x) {
+
+;--- count.ll
+define i32 @count(i32 %x) !prof !0 {
+entry:
+ switch i32 %x, label %def [
+ i32 42, label %def
+ i32 0, label %a
+ ], !prof !1
+def:
+ ret i32 0
+a:
+ ret i32 1
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 3000000000, i32 3000000000, i32 1}
+
+;--- expected.ll
+define i32 @expected(i32 %x) {
+entry:
+ switch i32 %x, label %def [
+ i32 42, label %def
+ i32 0, label %a
+ ], !prof !0
+def:
+ ret i32 0
+a:
+ ret i32 1
+}
+
+!0 = !{!"branch_weights", !"expected", i32 3000000000, i32 3000000000, i32 1}
diff --git a/llvm/unittests/IR/AttributesTest.cpp b/llvm/unittests/IR/AttributesTest.cpp
index 5e795ef771713..29d4463c7b13b 100644
--- a/llvm/unittests/IR/AttributesTest.cpp
+++ b/llvm/unittests/IR/AttributesTest.cpp
@@ -495,6 +495,7 @@ TEST(Attributes, SetIntersect) {
bool CanDrop = Attribute::intersectWithAnd(Kind) ||
Attribute::intersectWithMin(Kind) ||
Attribute::intersectWithCustom(Kind);
+ bool StickyOr = Attribute::intersectWithOr(Kind);
AB0.addAttribute(Attr0);
AB1.addAttribute(Attr1);
@@ -505,9 +506,13 @@ TEST(Attributes, SetIntersect) {
AS0 = AttributeSet::get(C0, AB0);
Res = AS0.intersectWith(C0, AS1);
- ASSERT_EQ(Res.has_value(), CanDrop);
+ ASSERT_EQ(Res.has_value(), CanDrop || StickyOr);
if (CanDrop)
ASSERT_FALSE(Res->hasAttributes());
+ else if (StickyOr) {
+ ASSERT_TRUE(Res->hasAttributes());
+ ASSERT_TRUE(Res->hasAttribute(Kind));
+ }
AS1 = AttributeSet::get(C1, AB0);
Res = AS0.intersectWith(C0, AS1);
@@ -516,7 +521,7 @@ TEST(Attributes, SetIntersect) {
AS1 = AttributeSet::get(C1, AB1);
Res = AS0.intersectWith(C0, AS1);
- if (!CanDrop) {
+ if (!CanDrop && !StickyOr) {
ASSERT_FALSE(Res.has_value());
continue;
}
@@ -530,6 +535,10 @@ TEST(Attributes, SetIntersect) {
ASSERT_TRUE(Res->hasAttributes());
ASSERT_TRUE(Res->hasAttribute(Kind));
ASSERT_FALSE(Res->hasAttribute(Other));
+ } else if (Attribute::intersectWithOr(Kind)) {
+ ASSERT_TRUE(Res.has_value());
+ ASSERT_TRUE(Res->hasAttributes());
+ ASSERT_TRUE(Res->hasAttribute(Kind));
} else if (Attribute::intersectWithMin(Kind)) {
ASSERT_TRUE(Res.has_value());
ASSERT_TRUE(Res->hasAttributes());
diff --git a/llvm/utils/TableGen/Basic/Attributes.cpp b/llvm/utils/TableGen/Basic/Attributes.cpp
index 66ba25c6dcc87..5bea1d70f101d 100644
--- a/llvm/utils/TableGen/Basic/Attributes.cpp
+++ b/llvm/utils/TableGen/Basic/Attributes.cpp
@@ -119,6 +119,7 @@ void Attributes::emitAttributeProperties(raw_ostream &OS) {
"ConstantRangeAttr", "ConstantRangeListAttr"}) {
bool AllowIntersectAnd = KindName == "EnumAttr";
bool AllowIntersectMin = KindName == "IntAttr";
+ bool AllowIntersectOr = KindName == "EnumAttr";
for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
OS << "0";
for (const Init *P : *A->getValueAsListInit("Properties")) {
@@ -128,6 +129,9 @@ void Attributes::emitAttributeProperties(raw_ostream &OS) {
if (!AllowIntersectMin &&
cast<DefInit>(P)->getDef()->getName() == "IntersectMin")
PrintFatalError("'IntersectMin' only compatible with 'IntAttr'");
+ if (!AllowIntersectOr &&
+ cast<DefInit>(P)->getDef()->getName() == "IntersectOr")
+ PrintFatalError("'IntersectOr' only compatible with 'EnumAttr'");
OS << " | AttributeProperty::" << cast<DefInit>(P)->getDef()->getName();
}
>From bc400435f57d44e27ad118d6418088374f341dab Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:04:26 +0530
Subject: [PATCH 06/13] [PGOFlowVerify] Skip strict checks on approx or
overflowed counts
Do not report BlockFrequencyMismatch or EntryCountMismatch when the
function is approxprofile or a branch weight does not fit in uint32.
Emit ApproxProfileSkip / CountOverflowSkip instead. Do not cache
overflowed weights. Treat calls from those callers as unknown.
-verify-pgo-flow-fatal aborts only on the two mismatch remarks.
---
.../llvm/Transforms/IPO/PGOFlowVerify.h | 6 +
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 108 +++++++++++++++---
.../verify-pgo-flow-approx-profile.ll | 59 ++++++++++
.../verify-pgo-flow-count-overflow-skip.ll | 106 +++++++++++++++++
.../PGOFlowVerifier/verify-pgo-flow-fatal.ll | 5 +-
5 files changed, 264 insertions(+), 20 deletions(-)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-approx-profile.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-count-overflow-skip.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
index aea94bf5b14ba..0a10fb9cca280 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -17,6 +17,7 @@
#define LLVM_TRANSFORMS_IPO_PGOFLOWVERIFY_H
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/LazyCallGraph.h"
@@ -56,12 +57,17 @@ class PGOFlowVerifier {
bool hasInstrProfUseSummary(const Module *M) const;
bool shouldVerifyFunction(const Function *F) const;
+ bool hasApproximateProfile(const Function *F) const;
+ bool hasU32WeightOverflow(const Function *F) const;
+ bool skipStrictInstrProfChecks(const Function *F, bool EmitNote) const;
void computeBlockFrequencies(const Function *F);
void validateBlockFrequencies(const Function *F);
void validateEntryCountAgainstCallerSum(const Function *F);
const AllBlockFreqInfo *getCachedBlockFreqInfo(const Function *F) const;
DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
+ DenseSet<const Function *> FunctionsWithU32WeightOverflow;
+ mutable DenseSet<const Function *> EmittedSkipNotes;
};
/// Pipeline pass that runs the same walk as the `-verify-pgo-flow` hook.
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
index 1d45fb72184e2..78bbc40de4c4f 100644
--- a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -15,6 +15,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/BasicBlock.h"
@@ -36,8 +37,8 @@
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
+#include <limits>
#include <memory>
-#include <numeric>
#include <vector>
using namespace llvm;
@@ -52,14 +53,19 @@ static cl::opt<bool> VerifyPGOFlowPrintDiagnostics(
"verify-pgo-flow-print-diagnostics", cl::init(true), cl::Hidden,
cl::desc("Print verify-pgo-flow banners and findings to stderr"));
-static cl::opt<bool>
- VerifyPGOFlowFatal("verify-pgo-flow-fatal", cl::init(false), cl::Hidden,
- cl::desc("Abort after a flow-check finding"));
+static cl::opt<bool> VerifyPGOFlowFatal(
+ "verify-pgo-flow-fatal", cl::init(false), cl::Hidden,
+ cl::desc("Abort after a BlockFrequencyMismatch or EntryCountMismatch"));
static cl::list<std::string> VerifyPGOFlowFuncList(
"verify-pgo-flow-funcs", cl::Hidden, cl::CommaSeparated,
cl::desc("If non-empty, only verify these functions"));
+static bool isStrictMismatchRemark(StringRef RemarkName) {
+ return RemarkName == "BlockFrequencyMismatch" ||
+ RemarkName == "EntryCountMismatch";
+}
+
static void printVerifyBanner(StringRef PassName, bool Skipped) {
if (!VerifyPGOFlowPrintDiagnostics)
return;
@@ -77,7 +83,7 @@ static void emitPGOFlowDiagnostic(const Function *F, StringRef RemarkName,
if (VerifyPGOFlowPrintDiagnostics)
errs() << "PGOFlowVerify[" << RemarkName << "] " << F->getName() << ": "
<< Text << "\n";
- if (VerifyPGOFlowFatal)
+ if (VerifyPGOFlowFatal && isStrictMismatchRemark(RemarkName))
report_fatal_error(Twine("PGOFlowVerify[") + RemarkName + "] " +
F->getName() + ": " + Text,
/*gen_crash_diag=*/false);
@@ -123,32 +129,40 @@ void PGOFlowVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
}
void PGOFlowVerifier::invalidateFunctionFrequencyCache(IRUnitRef IR) {
+ auto DropFunction = [&](const Function *F) {
+ FunctionBlockFreqInfoCache.erase(F);
+ FunctionsWithU32WeightOverflow.erase(F);
+ };
if (isa<Module>(IR)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: clear block-freq cache (module)\n");
FunctionBlockFreqInfoCache.clear();
+ FunctionsWithU32WeightOverflow.clear();
+ EmittedSkipNotes.clear();
return;
}
if (const auto *F = dyn_cast<Function>(IR)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: drop block-freq cache for '"
<< F->getName() << "'\n");
- FunctionBlockFreqInfoCache.erase(F);
+ DropFunction(F);
return;
}
if (const auto *C = dyn_cast<LazyCallGraph::SCC>(IR)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: drop block-freq cache for SCC\n");
for (const LazyCallGraph::Node &N : *C)
- FunctionBlockFreqInfoCache.erase(&N.getFunction());
+ DropFunction(&N.getFunction());
return;
}
if (const auto *L = dyn_cast<Loop>(IR)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: drop block-freq cache for loop\n");
if (L->getHeader())
- FunctionBlockFreqInfoCache.erase(L->getHeader()->getParent());
+ DropFunction(L->getHeader()->getParent());
return;
}
LLVM_DEBUG(
dbgs() << "PGOFlowVerifier: clear block-freq cache (unhandled IR)\n");
FunctionBlockFreqInfoCache.clear();
+ FunctionsWithU32WeightOverflow.clear();
+ EmittedSkipNotes.clear();
}
void PGOFlowVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
@@ -192,6 +206,37 @@ bool PGOFlowVerifier::shouldVerifyFunction(const Function *F) const {
return Listed;
}
+bool PGOFlowVerifier::hasApproximateProfile(const Function *F) const {
+ return F && hasApproximateProfileCounts(*F);
+}
+
+bool PGOFlowVerifier::hasU32WeightOverflow(const Function *F) const {
+ return F && FunctionsWithU32WeightOverflow.contains(F);
+}
+
+bool PGOFlowVerifier::skipStrictInstrProfChecks(const Function *F,
+ bool EmitNote) const {
+ if (hasApproximateProfile(F)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip strict checks for '"
+ << F->getName() << "' (approxprofile)\n");
+ if (EmitNote && EmittedSkipNotes.insert(F).second)
+ emitPGOFlowDiagnostic(
+ F, "ApproxProfileSkip",
+ "skipping strict InstrProf verification (approxprofile)");
+ return true;
+ }
+ if (hasU32WeightOverflow(F)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip strict checks for '"
+ << F->getName() << "' (u32 weight overflow)\n");
+ if (EmitNote && EmittedSkipNotes.insert(F).second)
+ emitPGOFlowDiagnostic(
+ F, "CountOverflowSkip",
+ "skipping strict InstrProf verification (profile count overflow)");
+ return true;
+ }
+ return false;
+}
+
void PGOFlowVerifier::runAfterPass(const Module *M) {
if (!M)
return;
@@ -206,11 +251,14 @@ void PGOFlowVerifier::runAfterPass(const Module *M) {
if (F.isDeclaration())
continue;
computeBlockFrequencies(&F);
- if (shouldVerifyFunction(&F))
- validateBlockFrequencies(&F);
+ if (!shouldVerifyFunction(&F) ||
+ skipStrictInstrProfChecks(&F, /*EmitNote=*/true))
+ continue;
+ validateBlockFrequencies(&F);
}
for (const Function &F : *M) {
- if (!shouldVerifyFunction(&F))
+ if (!shouldVerifyFunction(&F) ||
+ skipStrictInstrProfChecks(&F, /*EmitNote=*/false))
continue;
validateEntryCountAgainstCallerSum(&F);
}
@@ -225,6 +273,8 @@ void PGOFlowVerifier::runAfterPass(const Function *F) {
return;
}
computeBlockFrequencies(F);
+ if (skipStrictInstrProfChecks(F, /*EmitNote=*/true))
+ return;
validateBlockFrequencies(F);
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '" << F->getName()
<< "' (function-unit walk; need module-wide caller BFI)\n");
@@ -332,7 +382,7 @@ void PGOFlowVerifier::computeBlockFrequencies(const Function *F) {
if (SuccInfo.NumUnknownIn > 0)
SuccInfo.NumUnknownIn--;
if (Add)
- SuccInfo.SumIn += Add;
+ SuccInfo.SumIn = SaturatingAdd(SuccInfo.SumIn, Add);
if (ShouldProcess(Succ))
Enqueue(Succ);
};
@@ -391,21 +441,32 @@ void PGOFlowVerifier::computeBlockFrequencies(const Function *F) {
<< F->getName() << "' block " << BB->getName() << "\n");
return;
}
- SmallVector<uint32_t, 8> Weights32;
- if (extractBranchWeights(*Term, Weights32) &&
- Weights32.size() == Term->getNumSuccessors()) {
+ if (MDNode *WeightMD = getValidBranchWeightMDNode(*Term)) {
// Outs already closed (weights applied while a backedge was unknown).
if (Info.NumUnknownOut == 0)
return;
// No live flow yet.
if (Info.SumIn == 0)
return;
- SmallVector<uint64_t, 8> Weights(Weights32.begin(), Weights32.end());
+ SmallVector<uint64_t, 8> Weights;
+ extractFromBranchWeightMD64(WeightMD, Weights);
+ if (Weights.size() != Term->getNumSuccessors())
+ return;
+ for (uint64_t W : Weights) {
+ if (W > std::numeric_limits<uint32_t>::max()) {
+ LLVM_DEBUG(dbgs()
+ << "PGOFlowVerifier: u32 weight overflow in '"
+ << F->getName() << "' block " << BB->getName() << "\n");
+ FunctionsWithU32WeightOverflow.insert(F);
+ return;
+ }
+ }
Info.NumUnknownOut = 0;
- Info.SumOut =
- std::accumulate(Weights.begin(), Weights.end(), uint64_t(0));
- for (unsigned I = 0, E = Term->getNumSuccessors(); I < E; ++I)
+ Info.SumOut = 0;
+ for (unsigned I = 0, E = Term->getNumSuccessors(); I < E; ++I) {
ReleaseEdge(Term->getSuccessor(I), Weights[I]);
+ Info.SumOut = SaturatingAdd(Info.SumOut, Weights[I]);
+ }
return;
}
if (Info.NumUnknownIn != 0)
@@ -515,6 +576,15 @@ void PGOFlowVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
<< "'\n");
return;
}
+ if (hasApproximateProfile(CallerFunc) || hasU32WeightOverflow(CallerFunc)) {
+ HasUnknownCallsiteCount = true;
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: unknown callsite for '"
+ << F->getName() << "' (caller '"
+ << CallerFunc->getName()
+ << "' is approxprofile or u32-overflow)\n");
+ return;
+ }
+
bool NonzeroEntry = BB == &CallerFunc->getEntryBlock() &&
CallerFunc->getEntryCount() &&
*CallerFunc->getEntryCount() != 0;
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-approx-profile.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-approx-profile.ll
new file mode 100644
index 0000000000000..d07dc0989e74d
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-approx-profile.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=DIAG
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-fatal \
+; RUN: -verify-pgo-flow-funcs=approx_bad -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=SKIP-FATAL
+;
+; approxprofile means counts may be scaled; do not report a hard mismatch.
+; Skip notes must not abort under -verify-pgo-flow-fatal.
+
+; SKIP-FATAL: PGOFlowVerify[ApproxProfileSkip] approx_bad:
+; SKIP-FATAL-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+
+; DIAG: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; DIAG: PGOFlowVerify[ApproxProfileSkip] approx_bad: skipping strict InstrProf verification (approxprofile)
+; DIAG-NOT: PGOFlowVerify[BlockFrequencyMismatch] approx_bad:
+; DIAG: PGOFlowVerify[BlockFrequencyMismatch] strict_bad: block entry: incoming=10 vs outgoing=9
+; DIAG-NOT: PGOFlowVerify[BlockFrequencyMismatch] approx_bad:
+; DIAG-NOT: PGOFlowVerify[ApproxProfileSkip] strict_bad:
+
+define i32 @approx_bad(i32 %x) approxprofile !prof !0 {
+entry:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !2
+
+then:
+ ret i32 1
+
+else:
+ ret i32 0
+}
+
+define i32 @strict_bad(i32 %x) !prof !0 {
+entry:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !2
+
+then:
+ ret i32 1
+
+else:
+ ret i32 0
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!2 = !{!"branch_weights", i32 7, i32 2}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 20}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 7}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 4}
+!18 = !{!"NumFunctions", i64 2}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-count-overflow-skip.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-count-overflow-skip.ll
new file mode 100644
index 0000000000000..553cb12e49e29
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-count-overflow-skip.ll
@@ -0,0 +1,106 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=DIAG
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-fatal \
+; RUN: -verify-pgo-flow-funcs=huge_weight -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=SKIP-FATAL
+;
+; Counts that cannot fit in uint32_t on a terminator are not compared
+; strictly. A wide i64 function_entry_count is not overflow. Mismatch
+; against u32 outs is still reported.
+; Overflowed callers must not charge leftover call !prof onto a callee.
+; Skip notes must not abort under -verify-pgo-flow-fatal.
+
+; DIAG: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; DIAG: PGOFlowVerify[BlockFrequencyMismatch] overflow_bad: block entry: incoming=4294967296 vs outgoing=9
+; DIAG-NOT: PGOFlowVerify[CountOverflowSkip] overflow_bad:
+; DIAG: PGOFlowVerify[CountOverflowSkip] huge_weight: skipping strict InstrProf verification (profile count overflow)
+; DIAG-NOT: PGOFlowVerify[BlockFrequencyMismatch] huge_weight:
+; DIAG: PGOFlowVerify[BlockFrequencyMismatch] small_bad: block entry: incoming=10 vs outgoing=9
+; DIAG: PGOFlowVerify[CountOverflowSkip] huge_loop: skipping strict InstrProf verification (profile count overflow)
+; DIAG-NOT: PGOFlowVerify[BlockFrequencyMismatch] huge_loop:
+; DIAG-NOT: PGOFlowVerify[BlockFrequencyMismatch] overflow_bad:
+; DIAG-NOT: PGOFlowVerify[CountOverflowSkip] small_bad:
+; DIAG-NOT: PGOFlowVerify[EntryCountMismatch] overflow_callee:
+
+; SKIP-FATAL: PGOFlowVerify[CountOverflowSkip] huge_weight:
+; SKIP-FATAL-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+
+define i32 @overflow_bad(i32 %x) !prof !0 {
+entry:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !2
+
+then:
+ ret i32 1
+
+else:
+ ret i32 0
+}
+
+define internal i32 @overflow_callee(i32 %x) !prof !4 {
+entry:
+ ret i32 %x
+}
+
+define i32 @huge_weight(i32 %x) !prof !1 {
+entry:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !3
+
+then:
+ %a = call i32 @overflow_callee(i32 %x), !prof !5
+ ret i32 %a
+
+else:
+ %b = call i32 @overflow_callee(i32 %x), !prof !5
+ ret i32 %b
+}
+
+define i32 @small_bad(i32 %x) !prof !1 {
+entry:
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %else, !prof !2
+
+then:
+ ret i32 1
+
+else:
+ ret i32 0
+}
+
+; Overflow on a loop header is observed even though the backedge pred is
+; still unknown when the header weights are applied.
+define void @huge_loop(i1 %c) !prof !1 {
+entry:
+ br label %header
+
+header:
+ br i1 %c, label %latch, label %exit, !prof !3
+
+latch:
+ br label %header
+
+exit:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 4294967296}
+!1 = !{!"function_entry_count", i64 10}
+!2 = !{!"branch_weights", i32 7, i32 2}
+!3 = !{!"branch_weights", i64 4294967296, i64 1}
+!4 = !{!"function_entry_count", i64 1}
+!5 = !{!"branch_weights", i32 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 4294967306}
+!14 = !{!"MaxCount", i64 4294967296}
+!15 = !{!"MaxInternalCount", i64 4294967296}
+!16 = !{!"MaxFunctionCount", i64 4294967296}
+!17 = !{!"NumCounts", i64 4}
+!18 = !{!"NumFunctions", i64 5}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 4294967296, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-fatal.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-fatal.ll
index cc39c174693d5..3680a142654b6 100644
--- a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-fatal.ll
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-fatal.ll
@@ -1,7 +1,10 @@
; RUN: opt < %s -passes=instcombine -verify-pgo-flow \
; RUN: -verify-pgo-flow-fatal -disable-output 2>&1 | FileCheck %s
;
-; No findings yet, so -verify-pgo-flow-fatal must not abort.
+; No InstrProf summary and no mismatch, so -verify-pgo-flow-fatal must not abort.
+; Skip notes are covered in verify-pgo-flow-approx-profile.ll and
+; verify-pgo-flow-count-overflow-skip.ll, a real mismatch abort is in
+; verify-pgo-flow-block-frequency.ll.
; CHECK: *** PGO Flow Verification After InstCombinePass ***{{$}}
>From 44a0f234eb294a207d3596beedf5628f4c2b20de Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:04:40 +0530
Subject: [PATCH 07/13] [LoopUnroll] Mark approxprofile after peel and unroll
Peel, unroll, and unroll-and-jam rewrite count-type latch weights as
2^31-scaled probabilities. Stamp approxprofile so later checks do not
treat those weights as execution counts. Keep llvm.expect and unknown
origin on rewritten branches; do not stamp those functions.
---
.../include/llvm/Transforms/Utils/LoopUtils.h | 36 ++++-
llvm/lib/Transforms/Utils/LoopPeel.cpp | 21 ++-
.../Transforms/Utils/LoopUnrollRuntime.cpp | 74 +++++-----
llvm/lib/Transforms/Utils/LoopUtils.cpp | 78 +++++++++-
.../LoopUnroll/estimated-trip-count.ll | 12 +-
.../LoopUnroll/peel-branch-weights.ll | 1 +
.../peel-last-iteration-expansion-cost.ll | 3 +-
.../runtime-unroll-expected-epilog-prof.ll | 30 ++++
.../runtime-unroll-unknown-epilog-prof.ll | 56 ++++++++
.../LoopUnroll/zeroed-branch-weights.ll | 2 +-
.../LoopUnrollAndJam/approxprofile.ll | 38 +++++
.../verify-pgo-flow-unroll-peel-expected.ll | 134 ++++++++++++++++++
.../verify-pgo-flow-unroll-sibling-loop.ll | 54 +++++++
13 files changed, 488 insertions(+), 51 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-unroll-expected-epilog-prof.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-unroll-unknown-epilog-prof.ll
create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/approxprofile.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-peel-expected.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-sibling-loop.ll
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 74c549be35ddf..2090407db3ef6 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -13,9 +13,12 @@
#ifndef LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
#define LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
+#include <optional>
namespace llvm {
@@ -43,6 +46,7 @@ class SCEVExpander;
class TargetLibraryInfo;
class LPPassManager;
class Instruction;
+class MDNode;
struct RuntimeCheckingPtrGroup;
typedef std::pair<const RuntimeCheckingPtrGroup *,
const RuntimeCheckingPtrGroup *>
@@ -432,11 +436,41 @@ LLVM_ABI BranchProbability getBranchProbability(CondBrInst *B,
LLVM_ABI BranchProbability getBranchProbability(BasicBlock *Src,
BasicBlock *Dst);
+/// Set \p Weights on \p B, keeping the profile origin of \p Source: unknown
+/// stays unknown, llvm.expect stays expected. Does not mark \c approxprofile.
+LLVM_ABI void setBranchWeightsPreservingOrigin(Instruction &B,
+ ArrayRef<uint32_t> Weights,
+ const Instruction &Source);
+LLVM_ABI void setBranchWeightsPreservingOrigin(Instruction &B,
+ ArrayRef<uint32_t> Weights,
+ const MDNode *SourceMD);
+
+/// For new remainder/unroll guards. Do not copy llvm.expect; mark unknown with
+/// \p PassName. Count-type origin marks the function \c approxprofile.
+LLVM_ABI void setBranchWeightsForNewCFG(Instruction &B,
+ ArrayRef<uint32_t> Weights,
+ const Instruction &Source,
+ StringRef PassName);
+LLVM_ABI void setBranchWeightsForNewCFG(Instruction &B,
+ ArrayRef<uint32_t> Weights,
+ const MDNode *SourceMD,
+ StringRef PassName);
+
+/// Like \c setBranchProbability, but llvm.expect becomes unknown (\p PassName).
+LLVM_ABI void setBranchProbabilityForNewCFG(CondBrInst *B, BranchProbability P,
+ bool ForFirstTarget,
+ const Instruction *Origin,
+ StringRef PassName);
+
/// Set branch weight metadata for \p B to indicate that \p P and `1 - P` are
/// the probabilities of control flowing to its first and second target labels,
/// respectively, or vice-versa if \p ForFirstTarget is false.
+///
+/// These are probabilities, not counts. A count-type origin marks the function
+/// \c approxprofile.
LLVM_ABI void setBranchProbability(CondBrInst *B, BranchProbability P,
- bool ForFirstTarget);
+ bool ForFirstTarget,
+ const Instruction *Origin = nullptr);
/// Check inner loop (L) backedge count is known to be invariant on all
/// iterations of its outer loop. If the loop has no parent, this is trivially
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index 315763d442786..98ebae65fb8be 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -12,6 +12,7 @@
#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/Loads.h"
@@ -1242,8 +1243,12 @@ void llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
// probability of reaching iteration 0 of the original loop.
if (L->getExitBlock() == OrigLatchBr->getSuccessor(0))
std::swap(Weights[0], Weights[1]);
- setBranchWeights(*BI, Weights, /*IsExpected=*/false);
- }
+ // New CFG: keep the ratio, do not treat the integers as conserved
+ // counts.
+ setBranchWeightsForNewCFG(*BI, Weights, *OrigLatchBr, DEBUG_TYPE);
+ } else if (!ProfcheckDisableMetadataFixes &&
+ hasExplicitlyUnknownBranchWeights(*OrigLatchBr))
+ setExplicitlyUnknownBranchWeights(*BI, DEBUG_TYPE);
PreHeaderBR->eraseFromParent();
// PreHeader now dominates InsertTop.
@@ -1424,6 +1429,18 @@ void llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
setLoopEstimatedTripCount(L, EstimatedTripCountNew);
}
+ // Peel-first clones count-type branch weights onto a new CFG. Keep the
+ // integers for BPI; they are no longer conserved counts.
+ if (!PeelLast && !ProfcheckDisableMetadataFixes) {
+ for (BasicBlock *BB : L->blocks()) {
+ const MDNode *MD = BB->getTerminator()->getMetadata(LLVMContext::MD_prof);
+ if (isBranchWeightMD(MD) && !hasBranchWeightOrigin(MD)) {
+ markApproximateProfileCounts(*F);
+ break;
+ }
+ }
+ }
+
if (Loop *ParentLoop = L->getParentLoop())
L = ParentLoop;
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 92776cd1472bf..67455b0009e8a 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -28,7 +28,6 @@
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Dominators.h"
-#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/Support/CommandLine.h"
@@ -179,14 +178,13 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count,
SplitBlockPredecessors(OriginalLoopLatchExit, Preds, ".unr-lcssa", DT, LI,
nullptr, PreserveLCSSA);
// Add the branch to the exit block (around the unrolled loop)
- MDNode *BranchWeights = nullptr;
- if (hasBranchWeightMD(*Latch->getTerminator())) {
- // Assume loop is nearly always entered.
- MDBuilder MDB(B.getContext());
- BranchWeights = MDB.createBranchWeights(UnrolledLoopHeaderWeights);
- }
- B.CreateCondBr(BrLoopExit, OriginalLoopLatchExit, NewPreHeader,
- BranchWeights);
+ CondBrInst *UnrolledLoopGuard =
+ B.CreateCondBr(BrLoopExit, OriginalLoopLatchExit, NewPreHeader);
+ Instruction *OriginalLatch = Latch->getTerminator();
+ if (hasBranchWeightMD(*OriginalLatch) ||
+ hasExplicitlyUnknownBranchWeights(*OriginalLatch))
+ setBranchWeightsForNewCFG(*UnrolledLoopGuard, UnrolledLoopHeaderWeights,
+ *OriginalLatch, DEBUG_TYPE);
InsertPt->eraseFromParent();
if (DT) {
auto *NewDom = DT->findNearestCommonDominator(OriginalLoopLatchExit,
@@ -368,19 +366,19 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit,
SplitBlockPredecessors(Exit, Preds, ".epilog-lcssa", DT, LI, nullptr,
PreserveLCSSA);
// Add the branch to the exit block (around the epilog loop)
- MDNode *BranchWeights = nullptr;
+ CondBrInst *RemainderLoopGuard =
+ B.CreateCondBr(BrLoopExit, EpilogPreHeader, Exit);
+ Instruction *OriginalLatch = Latch->getTerminator();
if (OriginalLoopProb.isUnknown() &&
- hasBranchWeightMD(*Latch->getTerminator())) {
+ (hasBranchWeightMD(*OriginalLatch) ||
+ hasExplicitlyUnknownBranchWeights(*OriginalLatch)))
// Assume equal distribution in interval [0, Count).
- MDBuilder MDB(B.getContext());
- BranchWeights = MDB.createBranchWeights(1, Count - 1);
- }
- CondBrInst *RemainderLoopGuard =
- B.CreateCondBr(BrLoopExit, EpilogPreHeader, Exit, BranchWeights);
+ setBranchWeightsForNewCFG(*RemainderLoopGuard, {1, Count - 1},
+ *OriginalLatch, DEBUG_TYPE);
if (!OriginalLoopProb.isUnknown()) {
- setBranchProbability(RemainderLoopGuard,
- probOfNextInRemainder(OriginalLoopProb, Count - 1),
- /*ForFirstTarget=*/true);
+ setBranchProbabilityForNewCFG(
+ RemainderLoopGuard, probOfNextInRemainder(OriginalLoopProb, Count - 1),
+ /*ForFirstTarget=*/true, OriginalLatch, DEBUG_TYPE);
}
InsertPt->eraseFromParent();
if (DT) {
@@ -463,9 +461,11 @@ static Loop *CloneLoopBlocks(Loop *L, Value *NewIter,
Value *IdxNext =
Builder.CreateAdd(NewIdx, One, NewIdx->getName() + ".next");
Value *IdxCmp = Builder.CreateICmpNE(IdxNext, NewIter, NewIdx->getName() + ".cmp");
- MDNode *BranchWeights = nullptr;
+ SmallVector<uint32_t, 2> RemainderWeights;
+ Instruction *OriginalLatch = Latch->getTerminator();
if ((OriginalLoopProb.isUnknown() || !UseEpilogRemainder) &&
- hasBranchWeightMD(*LatchBR)) {
+ (hasBranchWeightMD(*OriginalLatch) ||
+ hasExplicitlyUnknownBranchWeights(*OriginalLatch))) {
uint32_t ExitWeight;
uint32_t BackEdgeWeight;
if (Count >= 3) {
@@ -480,11 +480,13 @@ static Loop *CloneLoopBlocks(Loop *L, Value *NewIter,
ExitWeight = 1;
BackEdgeWeight = 0;
}
- MDBuilder MDB(Builder.getContext());
- BranchWeights = MDB.createBranchWeights(BackEdgeWeight, ExitWeight);
+ RemainderWeights = {BackEdgeWeight, ExitWeight};
}
CondBrInst *RemainderLoopLatch =
- Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot, BranchWeights);
+ Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
+ if (!RemainderWeights.empty())
+ setBranchWeightsForNewCFG(*RemainderLoopLatch, RemainderWeights,
+ *OriginalLatch, DEBUG_TYPE);
if (!OriginalLoopProb.isUnknown() && UseEpilogRemainder) {
// Compute the total frequency of the original loop body from the
// remainder iterations. Once we've reached them, the first of them
@@ -501,7 +503,9 @@ static Loop *CloneLoopBlocks(Loop *L, Value *NewIter,
// Sum(i=0..inf)(Prob^i) = 1/(1-Prob) = FreqRemIters.
BranchProbability Prob =
BranchProbability::getBranchProbability(1 - 1 / FreqRemIters);
- setBranchProbability(RemainderLoopLatch, Prob, /*ForFirstTarget=*/true);
+ setBranchProbabilityForNewCFG(RemainderLoopLatch, Prob,
+ /*ForFirstTarget=*/true,
+ Latch->getTerminator(), DEBUG_TYPE);
}
NewIdx->addIncoming(Zero, InsertTop);
NewIdx->addIncoming(IdxNext, NewBB);
@@ -894,23 +898,23 @@ bool llvm::UnrollRuntimeLoopRemainder(
UseEpilogRemainder ? EpilogPreHeader : PrologPreHeader;
BasicBlock *UnrollingLoop = UseEpilogRemainder ? NewPreHeader : PrologExit;
// Branch to either remainder (extra iterations) loop or unrolling loop.
- MDNode *BranchWeights = nullptr;
- if ((OriginalLoopProb.isUnknown() || !UseEpilogRemainder) &&
- hasBranchWeightMD(*Latch->getTerminator())) {
- // Assume loop is nearly always entered.
- MDBuilder MDB(B.getContext());
- BranchWeights = MDB.createBranchWeights(EpilogHeaderWeights);
- }
+ Instruction *OriginalLatch = Latch->getTerminator();
CondBrInst *UnrollingLoopGuard =
- B.CreateCondBr(BranchVal, RemainderLoop, UnrollingLoop, BranchWeights);
+ B.CreateCondBr(BranchVal, RemainderLoop, UnrollingLoop);
+ if ((OriginalLoopProb.isUnknown() || !UseEpilogRemainder) &&
+ (hasBranchWeightMD(*OriginalLatch) ||
+ hasExplicitlyUnknownBranchWeights(*OriginalLatch)))
+ setBranchWeightsForNewCFG(*UnrollingLoopGuard, EpilogHeaderWeights,
+ *OriginalLatch, DEBUG_TYPE);
if (!OriginalLoopProb.isUnknown() && UseEpilogRemainder) {
// The original loop's first iteration always happens. Compute the
// probability of the original loop executing Count-1 iterations after that
// to complete the first iteration of the unrolled loop.
BranchProbability ProbOne = OriginalLoopProb;
BranchProbability ProbRest = ProbOne.pow(Count - 1);
- setBranchProbability(UnrollingLoopGuard, ProbRest,
- /*ForFirstTarget=*/false);
+ setBranchProbabilityForNewCFG(UnrollingLoopGuard, ProbRest,
+ /*ForFirstTarget=*/false,
+ Latch->getTerminator(), DEBUG_TYPE);
}
PreHeaderBR->eraseFromParent();
if (DT) {
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index a2e544801b9c4..0e5e4b366f663 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -32,8 +32,10 @@
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PatternMatch.h"
@@ -1003,9 +1005,9 @@ bool llvm::setLoopEstimatedTripCount(
if (LatchBranch->getSuccessor(0) != L->getHeader())
std::swap(BackedgeTakenWeight, LatchExitWeight);
- // Set/Update profile metadata.
- setBranchWeights(*LatchBranch, {BackedgeTakenWeight, LatchExitWeight},
- /*IsExpected=*/false);
+ // Keep the latch origin (expect / unknown / counts).
+ setBranchWeightsPreservingOrigin(
+ *LatchBranch, {BackedgeTakenWeight, LatchExitWeight}, *LatchBranch);
return true;
}
@@ -1074,14 +1076,78 @@ BranchProbability llvm::getBranchProbability(BasicBlock *Src, BasicBlock *Dst) {
return BranchProbability(Numerator, Total);
}
+static bool isCountTypeBranchWeights(const MDNode *MD) {
+ return isBranchWeightMD(MD) && !hasBranchWeightOrigin(MD);
+}
+
+static void markApproxIfCountTypeOrigin(Instruction &I,
+ const MDNode *OriginMD) {
+ if (!isCountTypeBranchWeights(OriginMD))
+ return;
+ if (Function *F = I.getFunction())
+ markApproximateProfileCounts(*F);
+}
+
+void llvm::setBranchWeightsPreservingOrigin(Instruction &B,
+ ArrayRef<uint32_t> Weights,
+ const Instruction &Source) {
+ setBranchWeightsPreservingOrigin(B, Weights,
+ Source.getMetadata(LLVMContext::MD_prof));
+}
+
+void llvm::setBranchWeightsPreservingOrigin(Instruction &B,
+ ArrayRef<uint32_t> Weights,
+ const MDNode *SourceMD) {
+ if (SourceMD && isExplicitlyUnknownProfileMetadata(*SourceMD)) {
+ B.setMetadata(LLVMContext::MD_prof, const_cast<MDNode *>(SourceMD));
+ return;
+ }
+
+ setBranchWeights(B, Weights, hasBranchWeightOrigin(SourceMD));
+}
+
void llvm::setBranchProbability(CondBrInst *B, BranchProbability P,
- bool ForFirstTarget) {
+ bool ForFirstTarget,
+ const Instruction *Origin) {
BranchProbability Prob0 = P;
BranchProbability Prob1 = P.getCompl();
if (!ForFirstTarget)
std::swap(Prob0, Prob1);
- setBranchWeights(*B, {Prob0.getNumerator(), Prob1.getNumerator()},
- /*IsExpected=*/false);
+
+ const Instruction &Src = Origin ? *Origin : *B;
+ const MDNode *OriginMD = Src.getMetadata(LLVMContext::MD_prof);
+ setBranchWeightsPreservingOrigin(
+ *B, {Prob0.getNumerator(), Prob1.getNumerator()}, OriginMD);
+ markApproxIfCountTypeOrigin(*B, OriginMD);
+}
+
+void llvm::setBranchWeightsForNewCFG(Instruction &B, ArrayRef<uint32_t> Weights,
+ const Instruction &Source,
+ StringRef PassName) {
+ setBranchWeightsForNewCFG(B, Weights,
+ Source.getMetadata(LLVMContext::MD_prof), PassName);
+}
+
+void llvm::setBranchWeightsForNewCFG(Instruction &B, ArrayRef<uint32_t> Weights,
+ const MDNode *SourceMD,
+ StringRef PassName) {
+ if (SourceMD && hasBranchWeightOrigin(SourceMD)) {
+ setExplicitlyUnknownBranchWeights(B, PassName);
+ return;
+ }
+ setBranchWeightsPreservingOrigin(B, Weights, SourceMD);
+ markApproxIfCountTypeOrigin(B, SourceMD);
+}
+
+void llvm::setBranchProbabilityForNewCFG(CondBrInst *B, BranchProbability P,
+ bool ForFirstTarget,
+ const Instruction *Origin,
+ StringRef PassName) {
+ if (Origin && hasBranchWeightOrigin(*Origin)) {
+ setExplicitlyUnknownBranchWeights(*B, PassName);
+ return;
+ }
+ setBranchProbability(B, P, ForFirstTarget, Origin);
}
bool llvm::hasIterationCountInvariantInParent(Loop *InnerLoop,
diff --git a/llvm/test/Transforms/LoopUnroll/estimated-trip-count.ll b/llvm/test/Transforms/LoopUnroll/estimated-trip-count.ll
index f822250026450..1c79ea3e98dc9 100644
--- a/llvm/test/Transforms/LoopUnroll/estimated-trip-count.ll
+++ b/llvm/test/Transforms/LoopUnroll/estimated-trip-count.ll
@@ -17,7 +17,7 @@ define void @zero_estimated_trip_count(ptr %p, i64 %n) !prof !0 {
; CHECK: [[EXIT]]:
;
; FORCED-LABEL: define void @zero_estimated_trip_count(
-; FORCED-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0:![0-9]+]] {
+; FORCED-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] !prof [[PROF0:![0-9]+]] {
; FORCED: [[ENTRY:.*:]]
; FORCED: br i1 [[TMP1:%.*]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]], !prof [[PROF1:![0-9]+]]
; FORCED: [[ENTRY_NEW]]:
@@ -54,7 +54,7 @@ exit:
; Same loop but with a high estimated trip count, which is runtime unrolled.
define void @high_estimated_trip_count(ptr %p, i64 %n) !prof !0 {
; CHECK-LABEL: define void @high_estimated_trip_count(
-; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] !prof [[PROF0]] {
; CHECK: [[ENTRY:.*:]]
; CHECK: br i1 [[TMP1:%.*]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]], !prof [[PROF4:![0-9]+]]
; CHECK: [[ENTRY_NEW]]:
@@ -72,7 +72,7 @@ define void @high_estimated_trip_count(ptr %p, i64 %n) !prof !0 {
; CHECK: [[EXIT]]:
;
; FORCED-LABEL: define void @high_estimated_trip_count(
-; FORCED-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; FORCED-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0]] !prof [[PROF0]] {
; FORCED: [[ENTRY:.*:]]
; FORCED: br i1 [[TMP1:%.*]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]], !prof [[PROF1]]
; FORCED: [[ENTRY_NEW]]:
@@ -110,7 +110,7 @@ exit:
; exit the loop.
define void @high_estimated_trip_count_low_branch_weights(ptr %p, i64 %n) !prof !0 {
; CHECK-LABEL: define void @high_estimated_trip_count_low_branch_weights(
-; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0]] !prof [[PROF0]] {
; CHECK: [[ENTRY:.*:]]
; CHECK: br i1 [[TMP1:%.*]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]], !prof [[PROF12:![0-9]+]]
; CHECK: [[ENTRY_NEW]]:
@@ -128,7 +128,7 @@ define void @high_estimated_trip_count_low_branch_weights(ptr %p, i64 %n) !prof
; CHECK: [[EXIT]]:
;
; FORCED-LABEL: define void @high_estimated_trip_count_low_branch_weights(
-; FORCED-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; FORCED-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0]] !prof [[PROF0]] {
; FORCED: [[ENTRY:.*:]]
; FORCED: br i1 [[TMP1:%.*]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]], !prof [[PROF12:![0-9]+]]
; FORCED: [[ENTRY_NEW]]:
@@ -170,6 +170,7 @@ exit:
!5 = !{!"llvm.loop.estimated_trip_count", i32 1024}
!6 = !{!"branch_weights", i32 1023, i32 1}
;.
+; CHECK: attributes #[[ATTR0]] = { approxprofile }
; CHECK: [[PROF0]] = !{!"function_entry_count", i64 1000}
; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1023}
; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
@@ -187,6 +188,7 @@ exit:
; CHECK: [[PROF14]] = !{!"branch_weights", i32 2097152, i32 2145386496}
; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META3]], [[META11]]}
;.
+; FORCED: attributes #[[ATTR0]] = { approxprofile }
; FORCED: [[PROF0]] = !{!"function_entry_count", i64 1000}
; FORCED: [[PROF1]] = !{!"branch_weights", i32 6285314, i32 2141198334}
; FORCED: [[PROF2]] = !{!"branch_weights", i32 8376328, i32 2139107320}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-branch-weights.ll b/llvm/test/Transforms/LoopUnroll/peel-branch-weights.ll
index 63a0dd4b4b4f9..fc0837d542b6c 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-branch-weights.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-branch-weights.ll
@@ -90,6 +90,7 @@ loop.exit:
!0 = !{!"branch_weights", i32 100, i32 200, i32 20, i32 10}
;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { approxprofile }
; CHECK: [[PROF0]] = !{!"branch_weights", i32 100, i32 200, i32 20, i32 10}
; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
; CHECK: [[META2]] = !{!"llvm.loop.peeled.count", i32 2}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-expansion-cost.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-expansion-cost.ll
index bc06625de0c76..0ec87482c8938 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-expansion-cost.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-expansion-cost.ll
@@ -26,7 +26,7 @@ define i32 @test_expansion_cost_2(i32 %start, i32 %end) !prof !0 {
; BUDGET2-NEXT: ret i32 0
;
; BUDGET3-LABEL: define i32 @test_expansion_cost_2(
-; BUDGET3-SAME: i32 [[START:%.*]], i32 [[END:%.*]]) !prof [[PROF0:![0-9]+]] {
+; BUDGET3-SAME: i32 [[START:%.*]], i32 [[END:%.*]]) #[[ATTR0:[0-9]+]] !prof [[PROF0:![0-9]+]] {
; BUDGET3-NEXT: [[ENTRY:.*]]:
; BUDGET3-NEXT: [[SUB:%.*]] = add i32 [[END]], -1
; BUDGET3-NEXT: [[TMP0:%.*]] = sub i32 [[SUB]], [[START]]
@@ -100,6 +100,7 @@ exit:
; BUDGET2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
; BUDGET2: [[META4]] = !{!"llvm.loop.estimated_trip_count", i32 42}
;.
+; BUDGET3: attributes #[[ATTR0]] = { approxprofile }
; BUDGET3: [[PROF0]] = !{!"function_entry_count", i32 10}
; BUDGET3: [[PROF1]] = !{!"branch_weights", i32 10, i32 1}
; BUDGET3: [[PROF2]] = !{!"branch_weights", i32 2, i32 3}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-expected-epilog-prof.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-expected-epilog-prof.ll
new file mode 100644
index 0000000000000..9151a52507020
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-expected-epilog-prof.ll
@@ -0,0 +1,30 @@
+; RUN: opt -passes=loop-unroll -unroll-runtime -unroll-count=4 -S %s \
+; RUN: | FileCheck %s
+;
+; Remainder guards are new edges. Don't copy llvm.expect onto them.
+
+; CHECK-NOT: Function Attrs: approxprofile
+; CHECK-LABEL: define void @expected_epilog(
+; CHECK: br i1 %lcmp.mod, label %loop.epil.preheader, label %exit, !prof [[UNKNOWN:![0-9]+]]
+; CHECK: [[UNKNOWN]] = !{!"unknown", !"loop-unroll"}
+
+define void @expected_epilog(ptr %p, i64 %n) !prof !1 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptr = getelementptr inbounds i32, ptr %p, i64 %iv
+ %v = load i32, ptr %ptr
+ %add = add i32 %v, 1
+ store i32 %add, ptr %ptr
+ %iv.next = add i64 %iv, 1
+ %cmp.loop = icmp eq i64 %iv.next, %n
+ br i1 %cmp.loop, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", !"expected", i32 1, i32 3}
+!1 = !{!"function_entry_count", i64 10}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-unknown-epilog-prof.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-unknown-epilog-prof.ll
new file mode 100644
index 0000000000000..a1da1fea4d481
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-unknown-epilog-prof.ll
@@ -0,0 +1,56 @@
+; RUN: opt -passes=loop-unroll -unroll-runtime -unroll-count=4 -S %s \
+; RUN: | FileCheck %s
+;
+; An unknown latch stays unknown on the guard; 0-0 counts do not.
+
+; CHECK-LABEL: define void @unknown_epilog(ptr %p, i64 %n) {
+; CHECK-NOT: Function Attrs:
+; CHECK: br i1 {{.*}}, label %loop.epil.preheader, label %entry.new, !prof [[UNKNOWN:![0-9]+]]
+; CHECK: br i1 %lcmp.mod, label %loop.epil.preheader, label %exit, !prof [[UNKNOWN]]
+;
+; CHECK: Function Attrs: approxprofile
+; CHECK-LABEL: define void @zero_zero_epilog(
+; CHECK: br i1 %lcmp.mod, label %loop.epil.preheader, label %exit, !prof [[ZERO_ZERO:![0-9]+]]
+; CHECK: attributes #[[ATTR:[0-9]+]] = { approxprofile }
+; CHECK: [[UNKNOWN]] = !{!"unknown", !"test"}
+; CHECK: [[ZERO_ZERO]] = !{!"branch_weights", i32 1, i32 3}
+
+define void @unknown_epilog(ptr %p, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptr = getelementptr inbounds i32, ptr %p, i64 %iv
+ %v = load i32, ptr %ptr
+ %add = add i32 %v, 1
+ store i32 %add, ptr %ptr
+ %iv.next = add i64 %iv, 1
+ %cmp.loop = icmp eq i64 %iv.next, %n
+ br i1 %cmp.loop, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+define void @zero_zero_epilog(ptr %p, i64 %n) !prof !1 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptr = getelementptr inbounds i32, ptr %p, i64 %iv
+ %v = load i32, ptr %ptr
+ %add = add i32 %v, 1
+ store i32 %add, ptr %ptr
+ %iv.next = add i64 %iv, 1
+ %cmp.loop = icmp eq i64 %iv.next, %n
+ br i1 %cmp.loop, label %exit, label %loop, !prof !2
+
+exit:
+ ret void
+}
+
+!0 = !{!"unknown", !"test"}
+!1 = !{!"function_entry_count", i64 10}
+!2 = !{!"branch_weights", i32 0, i32 0}
diff --git a/llvm/test/Transforms/LoopUnroll/zeroed-branch-weights.ll b/llvm/test/Transforms/LoopUnroll/zeroed-branch-weights.ll
index 4d378b0d22f7d..9ef7b72a200ef 100644
--- a/llvm/test/Transforms/LoopUnroll/zeroed-branch-weights.ll
+++ b/llvm/test/Transforms/LoopUnroll/zeroed-branch-weights.ll
@@ -16,7 +16,7 @@ end:
!0 = !{!"branch_weights", i32 0, i32 0}
-; CHECK: define void @test() {
+; CHECK-LABEL: define void @test() {
; CHECK: entry:
; CHECK: br label %loop
; CHECK: loop:
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/approxprofile.ll b/llvm/test/Transforms/LoopUnrollAndJam/approxprofile.ll
new file mode 100644
index 0000000000000..a8de55d6cc9c9
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/approxprofile.ll
@@ -0,0 +1,38 @@
+; RUN: opt -passes=loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -S < %s | FileCheck %s
+;
+; A constant trip count only clones the loop. Those counts stay counts.
+
+; CHECK-NOT: Function Attrs: approxprofile
+; CHECK: define void @jammed(
+
+define void @jammed(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {
+entry:
+ br label %for.outer
+
+for.outer:
+ %i = phi i32 [ %i.next, %for.latch ], [ 0, %entry ]
+ br label %for.inner
+
+for.inner:
+ %j = phi i32 [ 0, %for.outer ], [ %j.next, %for.inner ]
+ %sum = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+ %b.ptr = getelementptr inbounds i32, ptr %B, i32 %j
+ %b = load i32, ptr %b.ptr, align 4
+ %add = add i32 %b, %sum
+ %j.next = add nuw i32 %j, 1
+ %inner.exit = icmp eq i32 %j.next, 8
+ br i1 %inner.exit, label %for.latch, label %for.inner, !prof !0
+
+for.latch:
+ %sum.lcssa = phi i32 [ %add, %for.inner ]
+ %a.ptr = getelementptr inbounds i32, ptr %A, i32 %i
+ store i32 %sum.lcssa, ptr %a.ptr, align 4
+ %i.next = add nuw i32 %i, 1
+ %outer.exit = icmp eq i32 %i.next, 8
+ br i1 %outer.exit, label %exit, label %for.outer, !prof !0
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 7}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-peel-expected.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-peel-expected.ll
new file mode 100644
index 0000000000000..9f2dd43977a12
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-peel-expected.ll
@@ -0,0 +1,134 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu \
+; RUN: -passes='loop-unroll,verify-pgo-flow' -unroll-force-peel-count=2 \
+; RUN: -verify-pgo-flow-funcs=peel_expect \
+; RUN: -verify-pgo-flow-print-diagnostics=false -S \
+; RUN: | FileCheck %s --check-prefix=PEEL-IR --implicit-check-not=approxprofile
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu \
+; RUN: -passes='loop-unroll,verify-pgo-flow' -unroll-force-peel-count=2 \
+; RUN: -verify-pgo-flow-funcs=peel_expect -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=PEEL
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu \
+; RUN: -passes='loop-unroll,verify-pgo-flow' -unroll-runtime -unroll-count=2 \
+; RUN: -verify-pgo-flow-funcs=runtime_expect \
+; RUN: -verify-pgo-flow-print-diagnostics=false -S \
+; RUN: | FileCheck %s --check-prefix=RT-IR --implicit-check-not=approxprofile
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu \
+; RUN: -passes='loop-unroll,verify-pgo-flow' -unroll-runtime -unroll-count=2 \
+; RUN: -verify-pgo-flow-funcs=runtime_expect -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=RT
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes=loop-unroll \
+; RUN: -scev-cheap-expansion-budget=3 -S \
+; RUN: | FileCheck %s --check-prefix=PEEL-LAST-IR \
+; RUN: --implicit-check-not='!{!"branch_weights", i32' \
+; RUN: --implicit-check-not=approxprofile
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes=loop-unroll \
+; RUN: -unroll-runtime -unroll-runtime-epilog=false -unroll-count=2 -S \
+; RUN: | FileCheck %s --check-prefix=RT-PROLOG-IR \
+; RUN: --implicit-check-not='!{!"branch_weights", i32' \
+; RUN: --implicit-check-not=approxprofile
+;
+; llvm.expect is a hint, not a count. Peel-first and unroll keep !"expected".
+; Peel-last's BTC guard is new CFG, so expect becomes unknown. Never approxprofile.
+
+; PEEL: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; PEEL-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+; PEEL-NOT: PGOFlowVerify[ApproxProfileSkip]
+; PEEL-NOT: PGOFlowVerify[EntryCountMismatch]
+
+; PEEL-IR-LABEL: define void @peel_expect(i32 %n)
+; PEEL-IR-NOT: define void @peel_expect{{.*}}#
+; PEEL-IR: !{{[0-9]+}} = !{!"branch_weights", !"expected",
+
+; RT: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; RT-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+; RT-NOT: PGOFlowVerify[ApproxProfileSkip]
+; RT-NOT: PGOFlowVerify[EntryCountMismatch]
+
+; RT-IR-LABEL: define void @runtime_expect(i32 %n)
+; RT-IR-NOT: define void @runtime_expect{{.*}}#
+; RT-IR: !{{[0-9]+}} = !{!"branch_weights", !"expected",
+
+; PEEL-LAST-IR-LABEL: define i32 @peel_last_expect(
+; PEEL-LAST-IR: br i1 {{.*}}, label %{{.*}}, label %exit.peel.begin, !prof
+; PEEL-LAST-IR: exit.peel.begin:
+; PEEL-LAST-IR-NOT: !{!"branch_weights", i32
+; PEEL-LAST-IR: !{!"branch_weights", !"expected",
+; PEEL-LAST-IR: !{!"unknown", !"loop-peel"}
+
+; RT-PROLOG-IR-LABEL: define void @runtime_expect(
+; RT-PROLOG-IR: do.body.prol:
+; RT-PROLOG-IR-NOT: !{!"branch_weights", i32
+; RT-PROLOG-IR: !{!"branch_weights", !"expected",
+
+declare void @f(i32)
+
+define void @peel_expect(i32 %n) !prof !0 {
+entry:
+ br label %do.body
+
+do.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+ %inc = add i32 %i, 1
+ call void @f(i32 %i)
+ %c = icmp sge i32 %inc, %n
+ br i1 %c, label %do.end, label %do.body, !prof !1
+
+do.end:
+ ret void
+}
+
+define void @runtime_expect(i32 %n) !prof !0 {
+entry:
+ br label %do.body
+
+do.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+ %inc = add i32 %i, 1
+ call void @f(i32 %i)
+ %c = icmp sge i32 %inc, %n
+ br i1 %c, label %do.end, label %do.body, !prof !1
+
+do.end:
+ ret void
+}
+
+define i32 @peel_last_expect(i32 %start, i32 %end) !prof !0 {
+entry:
+ %sub = add i32 %end, -1
+ br label %loop.header
+
+loop.header:
+ %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop.latch ]
+ %c = icmp eq i32 %iv, %sub
+ br i1 %c, label %then, label %loop.latch, !prof !2
+
+then:
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %end
+ br i1 %ec, label %exit, label %loop.header, !prof !3
+
+exit:
+ ret i32 0
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", !"expected", i32 1, i32 2000}
+!2 = !{!"branch_weights", !"expected", i32 2, i32 3}
+!3 = !{!"branch_weights", !"expected", i32 1, i32 50}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 20}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 4}
+!18 = !{!"NumFunctions", i64 2}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-sibling-loop.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-sibling-loop.ll
new file mode 100644
index 0000000000000..c5a6d05e6772b
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-sibling-loop.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -passes='function(loop-unroll),verify-pgo-flow' \
+; RUN: -disable-output 2>&1 | FileCheck %s
+;
+; Unrolling A marks the function approxprofile, so B's BFI mismatch is skipped.
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK: PGOFlowVerify[ApproxProfileSkip] two_loops: skipping strict InstrProf verification (approxprofile)
+; CHECK-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+
+define void @two_loops(i32 %n, i32 %m) !prof !0 {
+entry:
+ br label %loop_a
+
+loop_a:
+ %ia = phi i32 [ 0, %entry ], [ %inca, %loop_a ]
+ %inca = add i32 %ia, 1
+ %ca = icmp sge i32 %inca, %n
+ br i1 %ca, label %loop_b.preheader, label %loop_a, !prof !1, !llvm.loop !10
+
+loop_b.preheader:
+ br label %loop_b
+
+loop_b:
+ %ib = phi i32 [ 0, %loop_b.preheader ], [ %incb, %loop_b ]
+ %incb = add i32 %ib, 1
+ %cb = icmp sge i32 %incb, %m
+ br i1 %cb, label %exit, label %loop_b, !prof !2, !llvm.loop !11
+
+exit:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 1, i32 9}
+!2 = !{!"branch_weights", i32 10, i32 1}
+
+!10 = distinct !{!10, !12}
+!11 = distinct !{!11, !13}
+!12 = !{!"llvm.loop.unroll.count", i32 2}
+!13 = !{!"llvm.loop.unroll.disable"}
+
+!llvm.module.flags = !{!20}
+!20 = !{i32 1, !"ProfileSummary", !21}
+!21 = !{!22, !23, !24, !25, !26, !27, !28, !29}
+!22 = !{!"ProfileFormat", !"InstrProf"}
+!23 = !{!"TotalCount", i64 20}
+!24 = !{!"MaxCount", i64 10}
+!25 = !{!"MaxInternalCount", i64 10}
+!26 = !{!"MaxFunctionCount", i64 10}
+!27 = !{!"NumCounts", i64 4}
+!28 = !{!"NumFunctions", i64 1}
+!29 = !{!"DetailedSummary", !30}
+!30 = !{!31}
+!31 = !{i32 10000, i64 10, i32 1}
>From 146dc568faf777f756e02712b979062c9165ad65 Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:05:10 +0530
Subject: [PATCH 08/13] [LV] Keep latch profile origin on estimated trip counts
LV writes new latch weights as counts that still conserve locally
(exit = W, backedge = (trip-1)*W). Do not stamp approxprofile for
that: stamp only when counts overflow or are converted to ratios.
Pass the original latch !prof so llvm.expect and unknown stay hints
on the new vector and remainder latches.
---
.../include/llvm/Transforms/Utils/LoopUtils.h | 5 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 10 +-
.../Vectorize/LoopVectorizationPlanner.h | 14 +--
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 8 +-
.../estimated-trip-count-profile-origin.ll | 95 +++++++++++++++++++
6 files changed, 124 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/estimated-trip-count-profile-origin.ll
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 2090407db3ef6..d0d3b2b3d2f13 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -396,9 +396,12 @@ getLoopEstimatedTripCount(Loop *L,
/// TODO: Eventually, once all passes have migrated away from setting branch
/// weights to indicate estimated trip counts, this function will drop the
/// \p EstimatedLoopInvocationWeight parameter.
+///
+/// For a new loop, pass the original latch \c !prof as \p ProfileOrigin.
LLVM_ABI bool setLoopEstimatedTripCount(
Loop *L, unsigned EstimatedTripCount,
- std::optional<unsigned> EstimatedLoopInvocationWeight = std::nullopt);
+ std::optional<unsigned> EstimatedLoopInvocationWeight = std::nullopt,
+ std::optional<const MDNode *> ProfileOrigin = std::nullopt);
/// Based on branch weight metadata, return either:
/// - An unknown probability if the implementation is unable to handle the loop
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 0e5e4b366f663..650a9fe6c5fd3 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -969,7 +969,8 @@ llvm::getLoopEstimatedTripCount(Loop *L,
bool llvm::setLoopEstimatedTripCount(
Loop *L, unsigned EstimatedTripCount,
- std::optional<unsigned> EstimatedloopInvocationWeight) {
+ std::optional<unsigned> EstimatedloopInvocationWeight,
+ std::optional<const MDNode *> ProfileOrigin) {
// If EstimatedLoopInvocationWeight, we do not support this loop if
// getExpectedExitLoopLatchBranch returns nullptr.
//
@@ -1005,9 +1006,12 @@ bool llvm::setLoopEstimatedTripCount(
if (LatchBranch->getSuccessor(0) != L->getHeader())
std::swap(BackedgeTakenWeight, LatchExitWeight);
- // Keep the latch origin (expect / unknown / counts).
+ // Keep origin from the original latch when L is newly created.
+ const MDNode *OriginMD = ProfileOrigin
+ ? *ProfileOrigin
+ : LatchBranch->getMetadata(LLVMContext::MD_prof);
setBranchWeightsPreservingOrigin(
- *LatchBranch, {BackedgeTakenWeight, LatchExitWeight}, *LatchBranch);
+ *LatchBranch, {BackedgeTakenWeight, LatchExitWeight}, OriginMD);
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 47311977dd379..a80a95dcf9bc5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -1024,15 +1024,17 @@ class LoopVectorizationPlanner {
/// Update loop metadata and profile info for both the scalar remainder loop
/// and \p VectorLoop, if it exists. Keeps all loop hints from the original
/// loop on the vector loop and replaces vectorizer-specific metadata. The
- /// loop ID of the original loop \p OrigLoopID must be passed, together with
- /// the average trip count and invocation weight of the original loop (\p
- /// OrigAverageTripCount and \p OrigLoopInvocationWeight respectively). They
- /// cannot be retrieved after the plan has been executed, as the original loop
- /// may have been removed. \p UnrollVectorizedLoop indicates whether the
- /// target wants the vector loop left eligible for runtime unrolling.
+ /// loop ID and latch profile of the original loop (\p OrigLoopID and
+ /// \p OrigLoopLatchProfile) must be passed, together with its average trip
+ /// count and invocation weight (\p OrigAverageTripCount and
+ /// \p OrigLoopInvocationWeight respectively). They cannot be retrieved after
+ /// the plan has been executed, as the original loop may have been removed.
+ /// \p UnrollVectorizedLoop indicates whether the target wants the vector loop
+ /// left eligible for runtime unrolling.
void updateLoopMetadataAndProfileInfo(
Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan,
bool VectorizingEpilogue, MDNode *OrigLoopID,
+ const MDNode *OrigLoopLatchProfile,
std::optional<unsigned> OrigAverageTripCount,
unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF,
bool DisableRuntimeUnroll, bool UnrollVectorizedLoop);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 82b1faaaa9b23..8f26dbf02868b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5843,6 +5843,10 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// Retrieve loop information before executing the plan, which may remove the
// original loop, if it becomes unreachable.
MDNode *LID = OrigLoop->getLoopID();
+ BasicBlock *OrigLatch = OrigLoop->getLoopLatch();
+ const MDNode *OrigLoopLatchProfile =
+ OrigLatch ? OrigLatch->getTerminator()->getMetadata(LLVMContext::MD_prof)
+ : nullptr;
unsigned OrigLoopInvocationWeight = 0;
std::optional<unsigned> OrigAverageTripCount =
getLoopEstimatedTripCount(OrigLoop, &OrigLoopInvocationWeight);
@@ -5862,7 +5866,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
: nullptr,
HeaderVPBB, BestVPlan,
EpilogueVecKind == EpilogueVectorizationKind::Epilogue, LID,
- OrigAverageTripCount, OrigLoopInvocationWeight,
+ OrigLoopLatchProfile, OrigAverageTripCount, OrigLoopInvocationWeight,
estimateElementCount(BestVF * BestUF, Config.getVScaleForTuning()),
DisableRuntimeUnroll, UnrollVectorizedLoop);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 812f46cee9dbb..277438bfd1b1c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1773,6 +1773,7 @@ static void addRuntimeUnrollDisableMetaData(Loop *L) {
void LoopVectorizationPlanner::updateLoopMetadataAndProfileInfo(
Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan,
bool VectorizingEpilogue, MDNode *OrigLoopID,
+ const MDNode *OrigLoopLatchProfile,
std::optional<unsigned> OrigAverageTripCount,
unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF,
bool DisableRuntimeUnroll, bool UnrollVectorizedLoop) {
@@ -1863,13 +1864,14 @@ void LoopVectorizationPlanner::updateLoopMetadataAndProfileInfo(
RemainderAverageTripCount = *OrigAverageTripCount % EstimatedVFxUF;
}
if (HeaderVPBB) {
- setLoopEstimatedTripCount(VectorLoop, AverageVectorTripCount,
- OrigLoopInvocationWeight);
+ if (VectorLoop->getLoopLatch())
+ setLoopEstimatedTripCount(VectorLoop, AverageVectorTripCount,
+ OrigLoopInvocationWeight, OrigLoopLatchProfile);
}
if (ScalarPH) {
setLoopEstimatedTripCount(OrigLoop, RemainderAverageTripCount,
- OrigLoopInvocationWeight);
+ OrigLoopInvocationWeight, OrigLoopLatchProfile);
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/estimated-trip-count-profile-origin.ll b/llvm/test/Transforms/LoopVectorize/estimated-trip-count-profile-origin.ll
new file mode 100644
index 0000000000000..68dfd9f4c923e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/estimated-trip-count-profile-origin.ll
@@ -0,0 +1,95 @@
+; RUN: split-file %s %t
+; RUN: opt < %t/count.ll -passes=loop-vectorize \
+; RUN: -loop-vectorize-with-block-frequency -force-vector-width=4 \
+; RUN: -force-vector-interleave=1 -S \
+; RUN: | FileCheck %s --check-prefix=COUNT \
+; RUN: --implicit-check-not=approxprofile
+; RUN: opt < %t/expected.ll -passes=loop-vectorize \
+; RUN: -loop-vectorize-with-block-frequency -force-vector-width=4 \
+; RUN: -force-vector-interleave=1 -S \
+; RUN: | FileCheck %s --check-prefix=EXPECTED \
+; RUN: --implicit-check-not=approxprofile
+; RUN: opt < %t/missing.ll -passes=loop-vectorize \
+; RUN: -loop-vectorize-with-block-frequency -force-vector-width=4 \
+; RUN: -force-vector-interleave=1 -S \
+; RUN: | FileCheck %s --check-prefix=MISSING \
+; RUN: --implicit-check-not=approxprofile
+;
+; The vector latch gets fresh weights that are still counts, so do not stamp.
+; llvm.expect stays a hint, and a latch with no !prof still gets an estimate.
+
+; COUNT-LABEL: define void @count(
+; COUNT: vector.body:
+; COUNT: br i1 {{.*}}, label %middle.block, label %vector.body, !prof [[COUNT_VEC:![0-9]+]]
+; COUNT: [[COUNT_VEC]] = !{!"branch_weights", i32 10, i32 2490}
+
+; EXPECTED-LABEL: define void @expected(
+; EXPECTED: vector.body:
+; EXPECTED: br i1 {{.*}}, label %middle.block, label %vector.body, !prof [[VECTOR_PROF:![0-9]+]]
+; EXPECTED: loop:
+; EXPECTED: br i1 {{.*}}, label %exit, label %loop, !prof [[SCALAR_PROF:![0-9]+]]
+; EXPECTED: [[VECTOR_PROF]] = !{!"branch_weights", !"expected", i32 10, i32 2490}
+; EXPECTED: [[SCALAR_PROF]] = !{!"branch_weights", !"expected",
+
+; MISSING-LABEL: define void @missing(
+; MISSING: vector.body:
+; MISSING: br i1 {{.*}}, label %middle.block, label %vector.body, !prof
+
+;--- count.ll
+define void @count(ptr %a, i32 %bound) !prof !0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ store i32 %iv, ptr %gep, align 4
+ %next = add nuw nsw i32 %iv, 1
+ %done = icmp eq i32 %next, %bound
+ br i1 %done, label %exit, label %loop, !prof !1
+
+exit:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 10, i32 10000}
+
+;--- expected.ll
+define void @expected(ptr %a, i32 %bound) !prof !0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ store i32 %iv, ptr %gep, align 4
+ %next = add nuw nsw i32 %iv, 1
+ %done = icmp eq i32 %next, %bound
+ br i1 %done, label %exit, label %loop, !prof !1
+
+exit:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", !"expected", i32 10, i32 10000}
+
+;--- missing.ll
+define void @missing(ptr %a) !prof !0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ store i32 %iv, ptr %gep, align 4
+ %next = add nuw nsw i32 %iv, 1
+ %done = icmp eq i32 %next, 16
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 10}
>From 56a467cc4954bb4b49f1f2386cd506dcf179333f Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:05:28 +0530
Subject: [PATCH 09/13] [PGOFlowVerify] Test peel and runtime unroll stay quiet
under approxprofile
Peel and runtime unroll stamp approxprofile. Check that verify-pgo-flow
emits ApproxProfileSkip, not BlockFrequencyMismatch, and that cloned
call !prof from an approxprofile caller is not charged onto a defined
callee.
---
.../verify-pgo-flow-unroll-peel-approx.ll | 181 ++++++++++++++++++
1 file changed, 181 insertions(+)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-peel-approx.ll
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-peel-approx.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-peel-approx.ll
new file mode 100644
index 0000000000000..95ef7fa99ef9d
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-unroll-peel-approx.ll
@@ -0,0 +1,181 @@
+; RUN: opt < %s -passes='loop-unroll,verify-pgo-flow' -unroll-force-peel-count=2 \
+; RUN: -verify-pgo-flow-funcs=peel_loop \
+; RUN: -verify-pgo-flow-print-diagnostics=false -S \
+; RUN: | FileCheck %s --check-prefix=PEEL-IR
+; RUN: opt < %s -passes='loop-unroll,verify-pgo-flow' -unroll-force-peel-count=2 \
+; RUN: -verify-pgo-flow-funcs=peel_loop -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=PEEL
+; RUN: opt < %s -passes='loop-unroll,verify-pgo-flow' -unroll-runtime -unroll-count=2 \
+; RUN: -verify-pgo-flow-funcs=runtime_loop \
+; RUN: -verify-pgo-flow-print-diagnostics=false -S \
+; RUN: | FileCheck %s --check-prefix=RT-IR
+; RUN: opt < %s -passes='loop-unroll,verify-pgo-flow' -unroll-runtime -unroll-count=2 \
+; RUN: -verify-pgo-flow-funcs=runtime_loop -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=RT
+; RUN: opt < %s -passes='function(loop-unroll),verify-pgo-flow' \
+; RUN: -unroll-force-peel-count=2 \
+; RUN: -verify-pgo-flow-funcs=peel_callee -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CALLEE-PEEL
+; RUN: opt < %s -passes='function(loop-unroll),verify-pgo-flow' \
+; RUN: -unroll-runtime -unroll-count=2 \
+; RUN: -verify-pgo-flow-funcs=peel_callee -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CALLEE-RT
+; RUN: opt < %s -passes=loop-unroll -unroll-force-peel-count=2 -S \
+; RUN: | FileCheck %s --check-prefix=CALL-IR
+; RUN: opt < %s -passes='loop-unroll,verify-pgo-flow' -unroll-force-peel-count=2 \
+; RUN: -verify-pgo-flow-funcs=peel_expect_vp -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=VP
+;
+; Peel-first clones count-type latch weights and stamps approxprofile.
+; Runtime unroll writes probabilities and stamps. Expect latches do not stamp.
+; Caller-sum runs on the module verifier after function(loop-unroll), not on a
+; function-unit verify-pgo-flow.
+
+declare void @f(i32)
+
+; PEEL: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; PEEL: PGOFlowVerify[ApproxProfileSkip] peel_loop: skipping strict InstrProf verification (approxprofile)
+; PEEL-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+; PEEL-NOT: PGOFlowVerify[EntryCountMismatch]
+
+; PEEL-IR-LABEL: define void @peel_loop(
+; PEEL-IR-SAME: #[[PEELATTR:[0-9]+]]
+; PEEL-IR: attributes #[[PEELATTR]] = { approxprofile }
+
+define void @peel_loop(i32 %n) !prof !0 {
+entry:
+ br label %do.body
+
+do.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+ %inc = add i32 %i, 1
+ call void @f(i32 %i)
+ %c = icmp sge i32 %inc, %n
+ br i1 %c, label %do.end, label %do.body, !prof !1
+
+do.end:
+ ret void
+}
+
+; RT: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; RT: PGOFlowVerify[ApproxProfileSkip] runtime_loop: skipping strict InstrProf verification (approxprofile)
+; RT-NOT: PGOFlowVerify[BlockFrequencyMismatch]
+; RT-NOT: PGOFlowVerify[EntryCountMismatch]
+
+; RT-IR-LABEL: define void @runtime_loop(
+; RT-IR-SAME: #[[RTATTR:[0-9]+]]
+; RT-IR: attributes #[[RTATTR]] = { approxprofile }
+
+define void @runtime_loop(i32 %n) !prof !0 {
+entry:
+ br label %do.body
+
+do.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+ %inc = add i32 %i, 1
+ call void @f(i32 %i)
+ %c = icmp sge i32 %inc, %n
+ br i1 %c, label %do.end, label %do.body, !prof !1
+
+do.end:
+ ret void
+}
+
+; CALLEE-PEEL: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CALLEE-PEEL-NOT: PGOFlowVerify[EntryCountMismatch] peel_callee:
+
+; CALLEE-RT: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CALLEE-RT-NOT: PGOFlowVerify[EntryCountMismatch] peel_callee:
+
+define internal void @peel_callee(i32 %x) !prof !30 {
+entry:
+ ret void
+}
+
+define void @peel_callee_loop(i32 %n) !prof !0 {
+entry:
+ br label %do.body
+
+do.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+ %inc = add i32 %i, 1
+ call void @peel_callee(i32 %i), !prof !31
+ %c = icmp sge i32 %inc, %n
+ br i1 %c, label %do.end, label %do.body, !prof !1
+
+do.end:
+ ret void
+}
+
+; CALL-IR-LABEL: define void @peel_call_count(i32 %n)
+; CALL-IR-NOT: define void @peel_call_count{{.*}}#
+
+; VP: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; VP-NOT: PGOFlowVerify[ApproxProfileSkip] peel_expect_vp:
+; VP-NOT: PGOFlowVerify[EntryCountMismatch]
+
+define void @peel_call_count(i32 %n) !prof !0 {
+entry:
+ br label %do.body
+
+do.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+ %inc = add i32 %i, 1
+ call void @f(i32 %i), !prof !31
+ %c = icmp sge i32 %inc, %n
+ br i1 %c, label %do.end, label %do.body, !prof !40
+
+do.end:
+ ret void
+}
+
+define void @peel_expect_vp(i32 %n) !prof !0 {
+entry:
+ br label %do.body
+
+do.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+ %inc = add i32 %i, 1
+ call void @f(i32 %i), !prof !41
+ %c = icmp sge i32 %inc, %n
+ br i1 %c, label %do.end, label %do.body, !prof !40
+
+do.end:
+ ret void
+}
+
+define void @runtime_callee_loop(i32 %n) !prof !0 {
+entry:
+ br label %do.body
+
+do.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+ %inc = add i32 %i, 1
+ call void @peel_callee(i32 %i), !prof !31
+ %c = icmp sge i32 %inc, %n
+ br i1 %c, label %do.end, label %do.body, !prof !1
+
+do.end:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 1, i32 9}
+!30 = !{!"function_entry_count", i64 1}
+!31 = !{!"branch_weights", i32 5}
+!40 = !{!"branch_weights", !"expected", i32 1, i32 9}
+!41 = !{!"VP", i32 0, i64 10, i64 123, i64 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 20}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 9}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 4}
+!18 = !{!"NumFunctions", i64 2}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
>From cdfae8d505baa1e677dcc8c1dd8b634a1c3664aa Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:05:43 +0530
Subject: [PATCH 10/13] [PGOFlowVerify] Report each inconsistent function once
The -verify-pgo-flow hook re-runs after every IR-changing pass, so one
real mismatch was printed over and over. Remember diagnosed functions
on the hook instance (hidden -verify-pgo-flow-dedup-diagnostics, on by
default) and skip those functions on later callbacks.
---
.../llvm/Transforms/IPO/PGOFlowVerify.h | 6 ++
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 32 +++++++++--
.../verify-pgo-flow-dedup-diagnostics.ll | 55 +++++++++++++++++++
3 files changed, 88 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dedup-diagnostics.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
index 0a10fb9cca280..0b98feda41865 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -20,6 +20,7 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/IR/IRUnitRef.h"
#include "llvm/IR/PassManager.h"
@@ -57,6 +58,9 @@ class PGOFlowVerifier {
bool hasInstrProfUseSummary(const Module *M) const;
bool shouldVerifyFunction(const Function *F) const;
+ bool shouldSkipReportedFunction(const Function *F) const;
+ void emitPGOFlowDiagnostic(const Function *F, StringRef RemarkName,
+ const Twine &Msg) const;
bool hasApproximateProfile(const Function *F) const;
bool hasU32WeightOverflow(const Function *F) const;
bool skipStrictInstrProfChecks(const Function *F, bool EmitNote) const;
@@ -68,6 +72,8 @@ class PGOFlowVerifier {
DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
DenseSet<const Function *> FunctionsWithU32WeightOverflow;
mutable DenseSet<const Function *> EmittedSkipNotes;
+ /// Real mismatches already printed when `-verify-pgo-flow-dedup-diagnostics`.
+ mutable DenseSet<const Function *> ReportedMismatchFunctions;
};
/// Pipeline pass that runs the same walk as the `-verify-pgo-flow` hook.
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
index 78bbc40de4c4f..465d96c861e5a 100644
--- a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -61,6 +61,11 @@ static cl::list<std::string> VerifyPGOFlowFuncList(
"verify-pgo-flow-funcs", cl::Hidden, cl::CommaSeparated,
cl::desc("If non-empty, only verify these functions"));
+static cl::opt<bool> VerifyPGOFlowDedupDiagnostics(
+ "verify-pgo-flow-dedup-diagnostics", cl::init(true), cl::Hidden,
+ cl::desc("Report each mismatch at most once and skip re-checking that "
+ "function on later passes"));
+
static bool isStrictMismatchRemark(StringRef RemarkName) {
return RemarkName == "BlockFrequencyMismatch" ||
RemarkName == "EntryCountMismatch";
@@ -75,10 +80,26 @@ static void printVerifyBanner(StringRef PassName, bool Skipped) {
bool PGOFlowVerifier::isHookEnabled() { return VerifyPGOFlow; }
-static void emitPGOFlowDiagnostic(const Function *F, StringRef RemarkName,
- const Twine &Msg) {
+bool PGOFlowVerifier::shouldSkipReportedFunction(const Function *F) const {
+ if (!VerifyPGOFlowDedupDiagnostics || !F)
+ return false;
+ if (!ReportedMismatchFunctions.contains(F))
+ return false;
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip '" << F->getName()
+ << "' (already reported mismatch)\n");
+ return true;
+}
+
+void PGOFlowVerifier::emitPGOFlowDiagnostic(const Function *F,
+ StringRef RemarkName,
+ const Twine &Msg) const {
if (!F)
return;
+ if (VerifyPGOFlowDedupDiagnostics && isStrictMismatchRemark(RemarkName)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: record mismatch '" << F->getName()
+ << "' [" << RemarkName << "]\n");
+ ReportedMismatchFunctions.insert(F);
+ }
std::string Text = Msg.str();
if (VerifyPGOFlowPrintDiagnostics)
errs() << "PGOFlowVerify[" << RemarkName << "] " << F->getName() << ": "
@@ -251,13 +272,13 @@ void PGOFlowVerifier::runAfterPass(const Module *M) {
if (F.isDeclaration())
continue;
computeBlockFrequencies(&F);
- if (!shouldVerifyFunction(&F) ||
+ if (!shouldVerifyFunction(&F) || shouldSkipReportedFunction(&F) ||
skipStrictInstrProfChecks(&F, /*EmitNote=*/true))
continue;
validateBlockFrequencies(&F);
}
for (const Function &F : *M) {
- if (!shouldVerifyFunction(&F) ||
+ if (!shouldVerifyFunction(&F) || shouldSkipReportedFunction(&F) ||
skipStrictInstrProfChecks(&F, /*EmitNote=*/false))
continue;
validateEntryCountAgainstCallerSum(&F);
@@ -265,7 +286,8 @@ void PGOFlowVerifier::runAfterPass(const Module *M) {
}
void PGOFlowVerifier::runAfterPass(const Function *F) {
- if (!F || !F->getParent() || !shouldVerifyFunction(F))
+ if (!F || !F->getParent() || !shouldVerifyFunction(F) ||
+ shouldSkipReportedFunction(F))
return;
if (!hasInstrProfUseSummary(F->getParent())) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip '" << F->getName()
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dedup-diagnostics.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dedup-diagnostics.ll
new file mode 100644
index 0000000000000..f7a7db7901c64
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dedup-diagnostics.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -verify-pgo-flow -passes='strip-dead-prototypes,globaldce' \
+; RUN: -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -verify-pgo-flow -verify-pgo-flow-dedup-diagnostics=false \
+; RUN: -passes='strip-dead-prototypes,globaldce' -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=REPEAT
+;
+; StripDeadPrototypes drops @dead_decl and GlobalDCE drops @dead_global, so
+; both module walks see a changed module and re-check @bad_callee's entry
+; count. Dedup (default) prints EntryCountMismatch once, disabling it prints
+; the same mismatch after every changing module walk.
+
+; CHECK: *** PGO Flow Verification After StripDeadPrototypesPass ***{{$}}
+; CHECK: PGOFlowVerify[EntryCountMismatch] bad_callee: entry=1 vs caller-sum=10
+; CHECK: *** PGO Flow Verification After GlobalDCEPass ***{{$}}
+; CHECK-NOT: PGOFlowVerify[EntryCountMismatch] bad_callee:
+
+; REPEAT: *** PGO Flow Verification After StripDeadPrototypesPass ***{{$}}
+; REPEAT: PGOFlowVerify[EntryCountMismatch] bad_callee: entry=1 vs caller-sum=10
+; REPEAT: *** PGO Flow Verification After GlobalDCEPass ***{{$}}
+; REPEAT: PGOFlowVerify[EntryCountMismatch] bad_callee: entry=1 vs caller-sum=10
+; REPEAT-NOT: PGOFlowVerify[EntryCountMismatch] bad_callee:
+
+define internal i32 @bad_callee(i32 %x) !prof !0 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define i32 @bad_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @bad_callee(i32 %x), !prof !2
+ ret i32 %r
+}
+
+ at dead_global = internal global i32 42
+
+declare void @dead_decl()
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 12}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 3}
+!18 = !{!"NumFunctions", i64 2}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
>From 08c106e2a2bd5faf2e6c0bac0d4ab2f62dc0e713 Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:05:56 +0530
Subject: [PATCH 11/13] [PGOFlowVerify] Make undercount and recursive
entry-count checks opt-in
Caller-sum > entry stays the default. Entry > sum and recursive callees
are easy false positives, so hide them behind flags and
-verify-pgo-flow-aggressive.
---
.../llvm/Transforms/IPO/PGOFlowVerify.h | 2 +
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 39 ++++++++--
.../verify-pgo-flow-entry-count.ll | 15 +++-
.../verify-pgo-flow-recursive.ll | 74 +++++++++++++++++++
4 files changed, 120 insertions(+), 10 deletions(-)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-recursive.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
index 0b98feda41865..372d5adda6683 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -66,6 +66,8 @@ class PGOFlowVerifier {
bool skipStrictInstrProfChecks(const Function *F, bool EmitNote) const;
void computeBlockFrequencies(const Function *F);
void validateBlockFrequencies(const Function *F);
+ /// Caller-sum > entry is always reported, including a live self-call.
+ /// Undercount and recursive undercount are opt-in.
void validateEntryCountAgainstCallerSum(const Function *F);
const AllBlockFreqInfo *getCachedBlockFreqInfo(const Function *F) const;
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
index 465d96c861e5a..bdaef76c2e7bd 100644
--- a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -66,6 +66,24 @@ static cl::opt<bool> VerifyPGOFlowDedupDiagnostics(
cl::desc("Report each mismatch at most once and skip re-checking that "
"function on later passes"));
+static cl::opt<bool> VerifyPGOFlowReportEntryCountUndercount(
+ "verify-pgo-flow-report-entry-count-undercount", cl::init(false),
+ cl::Hidden,
+ cl::desc("Report when entry count is higher than the visible caller-sum"));
+
+static cl::opt<bool> VerifyPGOFlowReportRecursiveEntryCountMismatch(
+ "verify-pgo-flow-report-recursive-entry-count-mismatch", cl::init(false),
+ cl::Hidden,
+ cl::desc("Report entry-count undercount on recursive functions"));
+
+static cl::opt<bool> VerifyPGOFlowAggressive(
+ "verify-pgo-flow-aggressive", cl::init(false), cl::Hidden,
+ cl::desc("Enable optional entry-count checks (undercount, recursive)"));
+
+static bool isEnabled(const cl::opt<bool> &Flag) {
+ return Flag || VerifyPGOFlowAggressive;
+}
+
static bool isStrictMismatchRemark(StringRef RemarkName) {
return RemarkName == "BlockFrequencyMismatch" ||
RemarkName == "EntryCountMismatch";
@@ -581,8 +599,6 @@ void PGOFlowVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
const Function *CallerFunc = BB->getParent();
if (!CallerFunc)
return;
- if (CallerFunc == F)
- IsRecursive = true;
const AllBlockFreqInfo *CallerFreq = getCachedBlockFreqInfo(CallerFunc);
if (!CallerFreq) {
@@ -630,6 +646,8 @@ void PGOFlowVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
<< "' block " << BB->getName() << "\n");
return;
}
+ if (CallerFunc == F)
+ IsRecursive = true;
HasAnyDirectCallsite = true;
Sum = SaturatingAdd(Sum, CallsiteCount);
};
@@ -658,9 +676,15 @@ void PGOFlowVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
<< F->getName() << "' (no direct callsite)\n");
return;
}
- if (IsRecursive) {
+ if (EntryCount == Sum) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
+ << F->getName() << "' (caller-sum equals entry)\n");
+ return;
+ }
+ if (Sum < EntryCount && !isEnabled(VerifyPGOFlowReportEntryCountUndercount)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
- << F->getName() << "' (recursive)\n");
+ << F->getName() << "' (undercount opt-in off, caller-sum="
+ << Sum << " < entry=" << EntryCount << ")\n");
return;
}
// Known Sum is a lower bound. Unknown sites can hide undercount, but not
@@ -670,10 +694,11 @@ void PGOFlowVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
<< F->getName() << "' (unknown callsite weight)\n");
return;
}
- if (Sum <= EntryCount) {
+ if (IsRecursive && Sum < EntryCount &&
+ !isEnabled(VerifyPGOFlowReportRecursiveEntryCountMismatch)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
- << F->getName() << "' (caller-sum=" << Sum
- << " <= entry=" << EntryCount << ")\n");
+ << F->getName()
+ << "' (recursive undercount opt-in off)\n");
return;
}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-entry-count.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-entry-count.ll
index ecb62c05e9779..b8af19e3c4655 100644
--- a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-entry-count.ll
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-entry-count.ll
@@ -4,10 +4,14 @@
; RUN: | FileCheck %s --check-prefix=FUNC
; RUN: not opt < %s -passes=verify-pgo-flow -verify-pgo-flow-fatal \
; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=FATAL
+; RUN: opt < %s -passes=verify-pgo-flow \
+; RUN: -verify-pgo-flow-report-entry-count-undercount \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=UNDER
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-aggressive \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=UNDER
;
-; Report only when visible direct-caller weight exceeds entry count.
-; Entry-count vs caller-sum is a module walk; function(verify-pgo-flow)
-; only checks in-function block flow.
+; Default: only caller-sum > entry. Function-unit walks skip that check.
+; UNDER also reports entry > caller-sum.
; DIAG: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
; DIAG-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
@@ -21,6 +25,11 @@
; FATAL: PGOFlowVerify[EntryCountMismatch]
+; UNDER-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
+; UNDER: PGOFlowVerify[EntryCountMismatch] undercount_callee: entry=10 vs caller-sum=1
+; UNDER: PGOFlowVerify[EntryCountMismatch] bad_callee: entry=1 vs caller-sum=10
+; UNDER-NOT: PGOFlowVerify[EntryCountMismatch] ok_callee:
+
define internal i32 @ok_callee(i32 %x) !prof !0 {
entry:
ret i32 %x
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-recursive.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-recursive.ll
new file mode 100644
index 0000000000000..0d96e55097c80
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-recursive.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=DEFAULT
+; RUN: opt < %s -passes=verify-pgo-flow \
+; RUN: -verify-pgo-flow-report-entry-count-undercount \
+; RUN: -verify-pgo-flow-report-recursive-entry-count-mismatch \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=REPORTED
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-aggressive \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=REPORTED
+;
+; The self-call is count-type !prof 3 vs entry 10. Silent by default;
+; reported only with the recursive/undercount opt-ins.
+; A dead leftover self-call must not hide a live external overcount.
+; A live self-call that overcounts is reported without the recursive opt-in.
+
+; DEFAULT: PGOFlowVerify[EntryCountMismatch] leftover_rec: entry=1 vs caller-sum=10
+; DEFAULT: PGOFlowVerify[EntryCountMismatch] rec_over: entry=1 vs caller-sum=10
+; DEFAULT-NOT: PGOFlowVerify[EntryCountMismatch] rec:
+
+; REPORTED: PGOFlowVerify[EntryCountMismatch] rec: entry=10 vs caller-sum=3
+
+define i32 @rec(i32 %n) !prof !0 {
+entry:
+ %cond = icmp sgt i32 %n, 0
+ br i1 %cond, label %recurse, label %base, !prof !1
+
+recurse:
+ %n1 = sub nsw i32 %n, 1
+ %r = call i32 @rec(i32 %n1), !prof !2
+ ret i32 %r
+
+base:
+ ret i32 0
+}
+
+define internal i32 @leftover_rec(i32 %x) !prof !3 {
+entry:
+ ret i32 %x
+
+dead:
+ %d = call i32 @leftover_rec(i32 %x), !prof !4
+ ret i32 %d
+}
+
+define i32 @leftover_rec_caller(i32 %x) !prof !3 {
+entry:
+ %r = call i32 @leftover_rec(i32 %x), !prof !4
+ ret i32 %r
+}
+
+define i32 @rec_over(i32 %n) !prof !3 {
+entry:
+ %r = call i32 @rec_over(i32 %n), !prof !4
+ ret i32 %r
+}
+
+!0 = !{!"function_entry_count", i64 10}
+!1 = !{!"branch_weights", i32 3, i32 7}
+!2 = !{!"branch_weights", i32 3}
+!3 = !{!"function_entry_count", i64 1}
+!4 = !{!"branch_weights", i32 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 10}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 3}
+!18 = !{!"NumFunctions", i64 1}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
>From 904c0ad86cc09ac165b883819cdd677ea958ef37 Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:06:09 +0530
Subject: [PATCH 12/13] [PGOFlowVerify] Credit value-profiled indirect calls in
the caller-sum
Off by default: stale VP data can inflate the sum and look like an
overcount. Match targets by PGO and IR-PGO name GUIDs.
-verify-pgo-flow-aggressive turns this on with the other optional
entry-count checks.
---
.../llvm/Transforms/IPO/PGOFlowVerify.h | 8 +-
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 126 +++++++++++++++++-
.../verify-pgo-flow-dead-indirect-callsite.ll | 48 +++++++
.../verify-pgo-flow-indirect-callers.ll | 83 ++++++++++++
4 files changed, 262 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dead-indirect-callsite.ll
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-indirect-callers.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
index 372d5adda6683..6daf8b0fd5bdb 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -67,8 +67,10 @@ class PGOFlowVerifier {
void computeBlockFrequencies(const Function *F);
void validateBlockFrequencies(const Function *F);
/// Caller-sum > entry is always reported, including a live self-call.
- /// Undercount and recursive undercount are opt-in.
+ /// Undercount, recursive undercount, and indirect credit are opt-in.
void validateEntryCountAgainstCallerSum(const Function *F);
+ uint64_t getIndirectCallTargetCount(const Function *F);
+ void updateIndirectCallTargetsForFunction(const Function *F);
const AllBlockFreqInfo *getCachedBlockFreqInfo(const Function *F) const;
DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
@@ -76,6 +78,10 @@ class PGOFlowVerifier {
mutable DenseSet<const Function *> EmittedSkipNotes;
/// Real mismatches already printed when `-verify-pgo-flow-dedup-diagnostics`.
mutable DenseSet<const Function *> ReportedMismatchFunctions;
+ DenseMap<uint64_t, uint64_t> IndirectCallTargetCounts;
+ DenseMap<const Function *, DenseMap<uint64_t, uint64_t>>
+ IndirectCallTargetContributionsByFunction;
+ bool IndirectCallTargetCountsValid = false;
};
/// Pipeline pass that runs the same walk as the `-verify-pgo-flow` hook.
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
index bdaef76c2e7bd..50c75b87c1645 100644
--- a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/IPO/PGOFlowVerify.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
+#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/LLVMContext.h"
@@ -32,6 +34,7 @@
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/ProfileSummary.h"
+#include "llvm/ProfileData/InstrProf.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
@@ -71,6 +74,10 @@ static cl::opt<bool> VerifyPGOFlowReportEntryCountUndercount(
cl::Hidden,
cl::desc("Report when entry count is higher than the visible caller-sum"));
+static cl::opt<bool> VerifyPGOFlowCreditIndirectCallers(
+ "verify-pgo-flow-credit-indirect-callers", cl::init(false), cl::Hidden,
+ cl::desc("Add value-profiled indirect calls to the caller-sum"));
+
static cl::opt<bool> VerifyPGOFlowReportRecursiveEntryCountMismatch(
"verify-pgo-flow-report-recursive-entry-count-mismatch", cl::init(false),
cl::Hidden,
@@ -78,7 +85,8 @@ static cl::opt<bool> VerifyPGOFlowReportRecursiveEntryCountMismatch(
static cl::opt<bool> VerifyPGOFlowAggressive(
"verify-pgo-flow-aggressive", cl::init(false), cl::Hidden,
- cl::desc("Enable optional entry-count checks (undercount, recursive)"));
+ cl::desc("Enable optional entry-count checks (undercount, indirect "
+ "credit, recursive)"));
static bool isEnabled(const cl::opt<bool> &Flag) {
return Flag || VerifyPGOFlowAggressive;
@@ -171,12 +179,17 @@ void PGOFlowVerifier::invalidateFunctionFrequencyCache(IRUnitRef IR) {
auto DropFunction = [&](const Function *F) {
FunctionBlockFreqInfoCache.erase(F);
FunctionsWithU32WeightOverflow.erase(F);
+ if (IndirectCallTargetCountsValid)
+ updateIndirectCallTargetsForFunction(F);
};
if (isa<Module>(IR)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: clear block-freq cache (module)\n");
FunctionBlockFreqInfoCache.clear();
FunctionsWithU32WeightOverflow.clear();
EmittedSkipNotes.clear();
+ IndirectCallTargetCounts.clear();
+ IndirectCallTargetContributionsByFunction.clear();
+ IndirectCallTargetCountsValid = false;
return;
}
if (const auto *F = dyn_cast<Function>(IR)) {
@@ -202,6 +215,9 @@ void PGOFlowVerifier::invalidateFunctionFrequencyCache(IRUnitRef IR) {
FunctionBlockFreqInfoCache.clear();
FunctionsWithU32WeightOverflow.clear();
EmittedSkipNotes.clear();
+ IndirectCallTargetCounts.clear();
+ IndirectCallTargetContributionsByFunction.clear();
+ IndirectCallTargetCountsValid = false;
}
void PGOFlowVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
@@ -575,6 +591,108 @@ PGOFlowVerifier::getCachedBlockFreqInfo(const Function *F) const {
return &It->second;
}
+void PGOFlowVerifier::updateIndirectCallTargetsForFunction(const Function *F) {
+ if (!F)
+ return;
+
+ auto OldIt = IndirectCallTargetContributionsByFunction.find(F);
+ if (OldIt != IndirectCallTargetContributionsByFunction.end()) {
+ for (const auto &Entry : OldIt->second) {
+ uint64_t &Total = IndirectCallTargetCounts[Entry.first];
+ Total = Total < Entry.second ? 0 : Total - Entry.second;
+ }
+ OldIt->second.clear();
+ }
+
+ DenseMap<uint64_t, uint64_t> &Contribution =
+ IndirectCallTargetContributionsByFunction[F];
+ if (F->isDeclaration() || hasApproximateProfile(F) ||
+ hasU32WeightOverflow(F)) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip indirect VP credit from '"
+ << F->getName()
+ << "' (declaration, approxprofile, or overflow)\n");
+ return;
+ }
+
+ const AllBlockFreqInfo *Freq = getCachedBlockFreqInfo(F);
+ if (!Freq) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip indirect VP credit from '"
+ << F->getName() << "' (no block-freq cache)\n");
+ return;
+ }
+ for (const BasicBlock &BB : *F) {
+ auto BBIt = Freq->find(&BB);
+ if (BBIt == Freq->end() || BBIt->second.NumUnknownIn != 0)
+ continue;
+ bool NonzeroEntry = &BB == &F->getEntryBlock() && F->getEntryCount() &&
+ *F->getEntryCount() != 0;
+ if (BBIt->second.SumIn == 0 && !NonzeroEntry) {
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip leftover VP in dead block "
+ << BB.getName() << " of '" << F->getName() << "'\n");
+ continue;
+ }
+ for (const Instruction &I : BB) {
+ const auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB || !CB->isIndirectCall())
+ continue;
+ uint64_t TotalC = 0;
+ auto VDs = getValueProfDataFromInst(*CB, IPVK_IndirectCallTarget,
+ std::numeric_limits<uint32_t>::max(),
+ TotalC);
+ if (TotalC == 0)
+ continue;
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: VP total " << TotalC << " on '"
+ << F->getName() << "'\n");
+ for (const InstrProfValueData &VD : VDs) {
+ Contribution[VD.Value] =
+ SaturatingAdd(Contribution[VD.Value], VD.Count);
+ IndirectCallTargetCounts[VD.Value] =
+ SaturatingAdd(IndirectCallTargetCounts[VD.Value], VD.Count);
+ }
+ }
+ }
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: refresh VP targets from '"
+ << F->getName() << "'\n");
+}
+
+uint64_t PGOFlowVerifier::getIndirectCallTargetCount(const Function *F) {
+ if (!F)
+ return 0;
+
+ if (!IndirectCallTargetCountsValid) {
+ LLVM_DEBUG(
+ dbgs() << "PGOFlowVerifier: build module indirect-call VP map\n");
+ IndirectCallTargetCounts.clear();
+ IndirectCallTargetContributionsByFunction.clear();
+ if (const Module *M = F->getParent()) {
+ for (const Function &Fn : *M)
+ updateIndirectCallTargetsForFunction(&Fn);
+ }
+ IndirectCallTargetCountsValid = true;
+ }
+
+ SmallDenseSet<uint64_t, 4> GUIDs;
+ auto InsertName = [&](const std::string &Name) {
+ if (!Name.empty())
+ GUIDs.insert(GlobalValue::getGUIDAssumingExternalLinkage(Name));
+ };
+ InsertName(getPGOFuncName(*F, /*InLTO=*/false));
+ InsertName(getPGOFuncName(*F, /*InLTO=*/true));
+ InsertName(getIRPGOFuncName(*F, /*InLTO=*/false));
+ InsertName(getIRPGOFuncName(*F, /*InLTO=*/true));
+
+ uint64_t Total = 0;
+ for (uint64_t GUID : GUIDs) {
+ auto It = IndirectCallTargetCounts.find(GUID);
+ if (It == IndirectCallTargetCounts.end())
+ continue;
+ Total = SaturatingAdd(Total, It->second);
+ }
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: indirect credit for '" << F->getName()
+ << "' is " << Total << "\n");
+ return Total;
+}
+
void PGOFlowVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
if (!F)
return;
@@ -671,11 +789,15 @@ void PGOFlowVerifier::validateEntryCountAgainstCallerSum(const Function *F) {
}
}
- if (!HasAnyDirectCallsite) {
+ uint64_t IndirectCredit = isEnabled(VerifyPGOFlowCreditIndirectCallers)
+ ? getIndirectCallTargetCount(F)
+ : 0;
+ if (!HasAnyDirectCallsite && IndirectCredit == 0) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
<< F->getName() << "' (no direct callsite)\n");
return;
}
+ Sum = SaturatingAdd(Sum, IndirectCredit);
if (EntryCount == Sum) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip entry-count for '"
<< F->getName() << "' (caller-sum equals entry)\n");
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dead-indirect-callsite.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dead-indirect-callsite.ll
new file mode 100644
index 0000000000000..7d3989524cc7f
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-dead-indirect-callsite.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -passes=verify-pgo-flow -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=verify-pgo-flow \
+; RUN: -verify-pgo-flow-credit-indirect-callers -disable-output 2>&1 \
+; RUN: | FileCheck %s
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-aggressive \
+; RUN: -disable-output 2>&1 | FileCheck %s
+;
+; Leftover VP on an unreachable block is not live indirect traffic. Do not
+; credit it into the callee's caller-sum (would be entry=1 vs caller-sum=101).
+
+; CHECK: *** PGO Flow Verification After verify-pgo-flow ***{{$}}
+; CHECK-NOT: PGOFlowVerify[EntryCountMismatch]
+
+define internal i32 @target(i32 %x) !prof !0 !PGOFuncName !5 {
+entry:
+ ret i32 %x
+}
+
+define i32 @caller(ptr %fp, i32 %x) !prof !1 {
+entry:
+ %r = call i32 @target(i32 %x), !prof !2
+ ret i32 %r
+
+dead:
+ %d = call i32 %fp(i32 %x), !prof !3
+ ret i32 %d
+}
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 1}
+; GUID of "target" is 15699497730709368386
+!3 = !{!"VP", i32 0, i64 100, i64 15699497730709368386, i64 100}
+!5 = !{!"target"}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 2}
+!14 = !{!"MaxCount", i64 1}
+!15 = !{!"MaxInternalCount", i64 1}
+!16 = !{!"MaxFunctionCount", i64 1}
+!17 = !{!"NumCounts", i64 3}
+!18 = !{!"NumFunctions", i64 2}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 1, i32 1}
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-indirect-callers.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-indirect-callers.ll
new file mode 100644
index 0000000000000..b163c259a3087
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-indirect-callers.ll
@@ -0,0 +1,83 @@
+; RUN: opt < %s -passes=verify-pgo-flow \
+; RUN: -verify-pgo-flow-report-entry-count-undercount \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=DIRECT
+; RUN: opt < %s -passes=verify-pgo-flow \
+; RUN: -verify-pgo-flow-report-entry-count-undercount \
+; RUN: -verify-pgo-flow-credit-indirect-callers -disable-output 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CREDIT
+; RUN: opt < %s -passes=verify-pgo-flow -verify-pgo-flow-aggressive \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=CREDIT
+;
+; Undercount on, no credit: mixed looks short (direct 6, entry 15).
+; With credit, VP 9 fills it. real_mismatch stays wrong (5+6 != 20).
+; callee_mixed is internal; VP GUID is MD5 of PGOFuncName "callee_mixed".
+
+; DIRECT: PGOFlowVerify[EntryCountMismatch] callee_mixed: entry=15 vs caller-sum=6
+; DIRECT: PGOFlowVerify[EntryCountMismatch] callee_real_mismatch: entry=20 vs caller-sum=5
+
+; CREDIT-NOT: PGOFlowVerify[EntryCountMismatch] callee_mixed:
+; CREDIT: PGOFlowVerify[EntryCountMismatch] callee_real_mismatch: entry=20 vs caller-sum=11
+; CREDIT-NOT: PGOFlowVerify[EntryCountMismatch] callee_mixed:
+
+define internal i32 @callee_mixed(i32 %x) !prof !0 !PGOFuncName !5 {
+entry:
+ ret i32 %x
+}
+
+define i32 @caller_direct_mixed(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @callee_mixed(i32 %x), !prof !2
+ ret i32 %r
+}
+
+define i32 @caller_indirect_mixed(ptr %fp, i32 %x) !prof !3 {
+entry:
+ %r = call i32 %fp(i32 %x), !prof !4
+ ret i32 %r
+}
+
+define i32 @callee_real_mismatch(i32 %x) !prof !10 {
+entry:
+ ret i32 %x
+}
+
+define i32 @caller_direct_real_mismatch(i32 %x) !prof !11 {
+entry:
+ %r = call i32 @callee_real_mismatch(i32 %x), !prof !12
+ ret i32 %r
+}
+
+define i32 @caller_indirect_real_mismatch(ptr %fp, i32 %x) !prof !13 {
+entry:
+ %r = call i32 %fp(i32 %x), !prof !14
+ ret i32 %r
+}
+
+!0 = !{!"function_entry_count", i64 15}
+!1 = !{!"function_entry_count", i64 6}
+!2 = !{!"branch_weights", i32 6}
+!3 = !{!"function_entry_count", i64 9}
+; GUID of "callee_mixed" is 5958130667041295651
+!4 = !{!"VP", i32 0, i64 9, i64 5958130667041295651, i64 9}
+!5 = !{!"callee_mixed"}
+
+!10 = !{!"function_entry_count", i64 20}
+!11 = !{!"function_entry_count", i64 5}
+!12 = !{!"branch_weights", i32 5}
+!13 = !{!"function_entry_count", i64 6}
+; GUID of "callee_real_mismatch" is 13528617892774658545
+!14 = !{!"VP", i32 0, i64 6, i64 13528617892774658545, i64 6}
+
+!llvm.module.flags = !{!100}
+!100 = !{i32 1, !"ProfileSummary", !101}
+!101 = !{!102, !103, !104, !105, !106, !107, !108, !109}
+!102 = !{!"ProfileFormat", !"InstrProf"}
+!103 = !{!"TotalCount", i64 50}
+!104 = !{!"MaxCount", i64 20}
+!105 = !{!"MaxInternalCount", i64 20}
+!106 = !{!"MaxFunctionCount", i64 20}
+!107 = !{!"NumCounts", i64 8}
+!108 = !{!"NumFunctions", i64 6}
+!109 = !{!"DetailedSummary", !110}
+!110 = !{!111}
+!111 = !{i32 10000, i64 20, i32 1}
>From b7b99b087d01406433820d003b340e8222dfd0fc Mon Sep 17 00:00:00 2001
From: Alok Kumar Sharma <AlokKumar.Sharma at amd.com>
Date: Wed, 9 Sep 2026 23:06:23 +0530
Subject: [PATCH 13/13] [PGOFlowVerify] Drop function-keyed caches when the
function is deleted
Watch cached functions with CallbackVH so block-frequency, VP credit, and
dedup maps cannot dangle or alias a new function at the same address.
---
.../llvm/Transforms/IPO/PGOFlowVerify.h | 19 +++++
llvm/lib/Transforms/IPO/PGOFlowVerify.cpp | 85 ++++++++++++++++---
.../verify-pgo-flow-deleted-function.ll | 47 ++++++++++
3 files changed, 137 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-deleted-function.ll
diff --git a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
index 6daf8b0fd5bdb..a1ec6bcf89ea9 100644
--- a/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
+++ b/llvm/include/llvm/Transforms/IPO/PGOFlowVerify.h
@@ -24,6 +24,7 @@
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/IR/IRUnitRef.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Compiler.h"
namespace llvm {
@@ -50,6 +51,17 @@ class PGOFlowVerifier {
LLVM_ABI void runAfterPass(StringRef PassID, IRUnitRef IR);
private:
+ class FunctionCallbackVH final : public CallbackVH {
+ PGOFlowVerifier *Parent = nullptr;
+ void deleted() override;
+ void allUsesReplacedWith(Value *) override;
+
+ public:
+ using DMI = DenseMapInfo<Value *>;
+ FunctionCallbackVH(Value *V, PGOFlowVerifier *Parent = nullptr)
+ : CallbackVH(V), Parent(Parent) {}
+ };
+
void invalidateFunctionFrequencyCache(IRUnitRef IR);
void runAfterPass(const Module *M);
void runAfterPass(const Function *F);
@@ -72,6 +84,10 @@ class PGOFlowVerifier {
uint64_t getIndirectCallTargetCount(const Function *F);
void updateIndirectCallTargetsForFunction(const Function *F);
const AllBlockFreqInfo *getCachedBlockFreqInfo(const Function *F) const;
+ void watchFunction(const Function *F) const;
+ void dropFunctionState(const Function *F);
+ void eraseFunctionHandle(Function *F);
+ void clearFunctionCaches();
DenseMap<const Function *, AllBlockFreqInfo> FunctionBlockFreqInfoCache;
DenseSet<const Function *> FunctionsWithU32WeightOverflow;
@@ -82,6 +98,9 @@ class PGOFlowVerifier {
DenseMap<const Function *, DenseMap<uint64_t, uint64_t>>
IndirectCallTargetContributionsByFunction;
bool IndirectCallTargetCountsValid = false;
+ /// Last so it is destroyed first while Function keys are still valid.
+ mutable DenseMap<FunctionCallbackVH, char, FunctionCallbackVH::DMI>
+ FunctionHandles;
};
/// Pipeline pass that runs the same walk as the `-verify-pgo-flow` hook.
diff --git a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
index 50c75b87c1645..497cd59776355 100644
--- a/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
+++ b/llvm/lib/Transforms/IPO/PGOFlowVerify.cpp
@@ -125,6 +125,7 @@ void PGOFlowVerifier::emitPGOFlowDiagnostic(const Function *F,
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: record mismatch '" << F->getName()
<< "' [" << RemarkName << "]\n");
ReportedMismatchFunctions.insert(F);
+ watchFunction(F);
}
std::string Text = Msg.str();
if (VerifyPGOFlowPrintDiagnostics)
@@ -175,6 +176,64 @@ void PGOFlowVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) {
});
}
+void PGOFlowVerifier::watchFunction(const Function *F) const {
+ if (!F)
+ return;
+ Function *MutF = const_cast<Function *>(F);
+ if (FunctionHandles.find_as(MutF) != FunctionHandles.end())
+ return;
+ FunctionHandles[FunctionCallbackVH(MutF,
+ const_cast<PGOFlowVerifier *>(this))] = 0;
+}
+
+void PGOFlowVerifier::eraseFunctionHandle(Function *F) {
+ auto It = FunctionHandles.find_as(F);
+ if (It != FunctionHandles.end())
+ FunctionHandles.erase(It);
+}
+
+void PGOFlowVerifier::dropFunctionState(const Function *F) {
+ if (!F)
+ return;
+ FunctionBlockFreqInfoCache.erase(F);
+ FunctionsWithU32WeightOverflow.erase(F);
+ EmittedSkipNotes.erase(F);
+ ReportedMismatchFunctions.erase(F);
+ auto OldIt = IndirectCallTargetContributionsByFunction.find(F);
+ if (OldIt != IndirectCallTargetContributionsByFunction.end()) {
+ for (const auto &Entry : OldIt->second) {
+ uint64_t &Total = IndirectCallTargetCounts[Entry.first];
+ Total = Total < Entry.second ? 0 : Total - Entry.second;
+ }
+ IndirectCallTargetContributionsByFunction.erase(OldIt);
+ }
+}
+
+void PGOFlowVerifier::clearFunctionCaches() {
+ FunctionBlockFreqInfoCache.clear();
+ FunctionsWithU32WeightOverflow.clear();
+ EmittedSkipNotes.clear();
+ IndirectCallTargetCounts.clear();
+ IndirectCallTargetContributionsByFunction.clear();
+ IndirectCallTargetCountsValid = false;
+}
+
+void PGOFlowVerifier::FunctionCallbackVH::deleted() {
+ Function *F = cast<Function>(getValPtr());
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: drop state for deleted '"
+ << F->getName() << "'\n");
+ Parent->dropFunctionState(F);
+ Parent->eraseFunctionHandle(F);
+}
+
+void PGOFlowVerifier::FunctionCallbackVH::allUsesReplacedWith(Value *) {
+ Function *F = cast<Function>(getValPtr());
+ LLVM_DEBUG(dbgs() << "PGOFlowVerifier: drop state for replaced '"
+ << F->getName() << "'\n");
+ Parent->dropFunctionState(F);
+ Parent->eraseFunctionHandle(F);
+}
+
void PGOFlowVerifier::invalidateFunctionFrequencyCache(IRUnitRef IR) {
auto DropFunction = [&](const Function *F) {
FunctionBlockFreqInfoCache.erase(F);
@@ -184,12 +243,7 @@ void PGOFlowVerifier::invalidateFunctionFrequencyCache(IRUnitRef IR) {
};
if (isa<Module>(IR)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: clear block-freq cache (module)\n");
- FunctionBlockFreqInfoCache.clear();
- FunctionsWithU32WeightOverflow.clear();
- EmittedSkipNotes.clear();
- IndirectCallTargetCounts.clear();
- IndirectCallTargetContributionsByFunction.clear();
- IndirectCallTargetCountsValid = false;
+ clearFunctionCaches();
return;
}
if (const auto *F = dyn_cast<Function>(IR)) {
@@ -212,12 +266,7 @@ void PGOFlowVerifier::invalidateFunctionFrequencyCache(IRUnitRef IR) {
}
LLVM_DEBUG(
dbgs() << "PGOFlowVerifier: clear block-freq cache (unhandled IR)\n");
- FunctionBlockFreqInfoCache.clear();
- FunctionsWithU32WeightOverflow.clear();
- EmittedSkipNotes.clear();
- IndirectCallTargetCounts.clear();
- IndirectCallTargetContributionsByFunction.clear();
- IndirectCallTargetCountsValid = false;
+ clearFunctionCaches();
}
void PGOFlowVerifier::runAfterPass(StringRef PassID, IRUnitRef IR) {
@@ -274,19 +323,23 @@ bool PGOFlowVerifier::skipStrictInstrProfChecks(const Function *F,
if (hasApproximateProfile(F)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip strict checks for '"
<< F->getName() << "' (approxprofile)\n");
- if (EmitNote && EmittedSkipNotes.insert(F).second)
+ if (EmitNote && EmittedSkipNotes.insert(F).second) {
+ watchFunction(F);
emitPGOFlowDiagnostic(
F, "ApproxProfileSkip",
"skipping strict InstrProf verification (approxprofile)");
+ }
return true;
}
if (hasU32WeightOverflow(F)) {
LLVM_DEBUG(dbgs() << "PGOFlowVerifier: skip strict checks for '"
<< F->getName() << "' (u32 weight overflow)\n");
- if (EmitNote && EmittedSkipNotes.insert(F).second)
+ if (EmitNote && EmittedSkipNotes.insert(F).second) {
+ watchFunction(F);
emitPGOFlowDiagnostic(
F, "CountOverflowSkip",
"skipping strict InstrProf verification (profile count overflow)");
+ }
return true;
}
return false;
@@ -371,6 +424,7 @@ void PGOFlowVerifier::computeBlockFrequencies(const Function *F) {
AllFreqInfo[&BB].SumOut = 0;
}
FunctionBlockFreqInfoCache[F] = std::move(AllFreqInfo);
+ watchFunction(F);
return;
} else if (const Instruction *EntryTerm =
F->getEntryBlock().getTerminator();
@@ -514,6 +568,7 @@ void PGOFlowVerifier::computeBlockFrequencies(const Function *F) {
<< "PGOFlowVerifier: u32 weight overflow in '"
<< F->getName() << "' block " << BB->getName() << "\n");
FunctionsWithU32WeightOverflow.insert(F);
+ watchFunction(F);
return;
}
}
@@ -546,6 +601,7 @@ void PGOFlowVerifier::computeBlockFrequencies(const Function *F) {
}
FunctionBlockFreqInfoCache[F] = std::move(AllFreqInfo);
+ watchFunction(F);
}
void PGOFlowVerifier::validateBlockFrequencies(const Function *F) {
@@ -594,6 +650,7 @@ PGOFlowVerifier::getCachedBlockFreqInfo(const Function *F) const {
void PGOFlowVerifier::updateIndirectCallTargetsForFunction(const Function *F) {
if (!F)
return;
+ watchFunction(F);
auto OldIt = IndirectCallTargetContributionsByFunction.find(F);
if (OldIt != IndirectCallTargetContributionsByFunction.end()) {
diff --git a/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-deleted-function.ll b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-deleted-function.ll
new file mode 100644
index 0000000000000..01f13afab77fe
--- /dev/null
+++ b/llvm/test/Transforms/PGOFlowVerifier/verify-pgo-flow-deleted-function.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -verify-pgo-flow \
+; RUN: -passes='function(instcombine),globaldce,function(instcombine)' \
+; RUN: -disable-output 2>&1 | FileCheck %s
+;
+; InstCombine caches unused @gone. GlobalDCE deletes it. CallbackVH must
+; drop function-keyed maps so the later InstCombine walk does not use a
+; dangling Function* (that typically crashes as UAF). @bad_callee must
+; still be diagnosed afterward.
+
+; CHECK: PGOFlowVerify[EntryCountMismatch] bad_callee: entry=1 vs caller-sum=10
+; CHECK-NOT: PGOFlowVerify{{.*}}gone
+
+define internal void @gone() !prof !0 {
+entry:
+ %a = add i32 0, 0
+ ret void
+}
+
+define internal i32 @bad_callee(i32 %x) !prof !0 {
+entry:
+ %y = add i32 %x, 0
+ ret i32 %y
+}
+
+define i32 @bad_caller(i32 %x) !prof !1 {
+entry:
+ %r = call i32 @bad_callee(i32 %x), !prof !2
+ ret i32 %r
+}
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 10}
+
+!llvm.module.flags = !{!10}
+!10 = !{i32 1, !"ProfileSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !{!"ProfileFormat", !"InstrProf"}
+!13 = !{!"TotalCount", i64 12}
+!14 = !{!"MaxCount", i64 10}
+!15 = !{!"MaxInternalCount", i64 10}
+!16 = !{!"MaxFunctionCount", i64 10}
+!17 = !{!"NumCounts", i64 3}
+!18 = !{!"NumFunctions", i64 3}
+!19 = !{!"DetailedSummary", !20}
+!20 = !{!21}
+!21 = !{i32 10000, i64 10, i32 1}
More information about the cfe-commits
mailing list