[llvm] [PGO] Add ProfileInjector and ProfileVerifier passes (PR #147388)
Mircea Trofin via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 21 08:12:08 PDT 2025
https://github.com/mtrofin updated https://github.com/llvm/llvm-project/pull/147388
>From 9200ff50231e706acbc02693c170fca772553a65 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin at google.com>
Date: Mon, 7 Jul 2025 12:44:03 -0700
Subject: [PATCH] pass validator
---
.../llvm/Transforms/Utils/ProfileVerify.h | 36 ++++++
llvm/lib/Passes/PassBuilder.cpp | 1 +
llvm/lib/Passes/PassRegistry.def | 2 +
llvm/lib/Transforms/Utils/CMakeLists.txt | 1 +
llvm/lib/Transforms/Utils/ProfileVerify.cpp | 111 ++++++++++++++++++
.../PGOProfile/prof-verify-as-needed.ll | 20 ++++
.../PGOProfile/prof-verify-existing.ll | 21 ++++
.../test/Transforms/PGOProfile/prof-verify.ll | 19 +++
8 files changed, 211 insertions(+)
create mode 100644 llvm/include/llvm/Transforms/Utils/ProfileVerify.h
create mode 100644 llvm/lib/Transforms/Utils/ProfileVerify.cpp
create mode 100644 llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll
create mode 100644 llvm/test/Transforms/PGOProfile/prof-verify-existing.ll
create mode 100644 llvm/test/Transforms/PGOProfile/prof-verify.ll
diff --git a/llvm/include/llvm/Transforms/Utils/ProfileVerify.h b/llvm/include/llvm/Transforms/Utils/ProfileVerify.h
new file mode 100644
index 0000000000000..88942a73474d4
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/ProfileVerify.h
@@ -0,0 +1,36 @@
+//===- ProfileVerify.h - Verify profile info for testing ----0-----*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Inject profile information, as part of tests, to verify passes don't
+// accidentally drop it.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_UTILS_PROFILEVERIFY_H
+#define LLVM_TRANSFORMS_UTILS_PROFILEVERIFY_H
+
+#include "llvm/IR/Analysis.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+/// Inject MD_prof metadata where it's missing. Used for testing that passes
+/// don't accidentally drop this metadata.
+class ProfileInjectorPass : public PassInfoMixin<ProfileInjectorPass> {
+public:
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+/// Checks that MD_prof is present on every instruction that supports it. Used
+/// in conjunction with the ProfileInjectorPass. MD_prof "unknown" is considered
+/// valid (i.e. !{!"unknown"})
+class ProfileVerifierPass : public PassInfoMixin<ProfileVerifierPass> {
+public:
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+} // namespace llvm
+#endif
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 874fce05841e2..70c0e999cbb95 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -357,6 +357,7 @@
#include "llvm/Transforms/Utils/MoveAutoInit.h"
#include "llvm/Transforms/Utils/NameAnonGlobals.h"
#include "llvm/Transforms/Utils/PredicateInfo.h"
+#include "llvm/Transforms/Utils/ProfileVerify.h"
#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
#include "llvm/Transforms/Utils/StripGCRelocates.h"
#include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index dd3dab3425975..dfe233f15a149 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -517,6 +517,8 @@ FUNCTION_PASS("print<regions>", RegionInfoPrinterPass(errs()))
FUNCTION_PASS("print<scalar-evolution>", ScalarEvolutionPrinterPass(errs()))
FUNCTION_PASS("print<stack-safety-local>", StackSafetyPrinterPass(errs()))
FUNCTION_PASS("print<uniformity>", UniformityInfoPrinterPass(errs()))
+FUNCTION_PASS("prof-inject", ProfileInjectorPass())
+FUNCTION_PASS("prof-verify", ProfileVerifierPass())
FUNCTION_PASS("reassociate", ReassociatePass())
FUNCTION_PASS("redundant-dbg-inst-elim", RedundantDbgInstEliminationPass())
FUNCTION_PASS("reg2mem", RegToMemPass())
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 78cad0d253be8..c0bd6d647aad4 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -67,6 +67,7 @@ add_llvm_component_library(LLVMTransformUtils
MoveAutoInit.cpp
NameAnonGlobals.cpp
PredicateInfo.cpp
+ ProfileVerify.cpp
PromoteMemoryToRegister.cpp
RelLookupTableConverter.cpp
ScalarEvolutionExpander.cpp
diff --git a/llvm/lib/Transforms/Utils/ProfileVerify.cpp b/llvm/lib/Transforms/Utils/ProfileVerify.cpp
new file mode 100644
index 0000000000000..e3b9df8c4262b
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/ProfileVerify.cpp
@@ -0,0 +1,111 @@
+//===- ProfileVerify.cpp - Verify profile info for testing ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/ProfileVerify.h"
+#include "llvm/ADT/DynamicAPInt.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Analysis.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/ProfDataUtils.h"
+#include "llvm/Support/BranchProbability.h"
+
+using namespace llvm;
+namespace {
+class ProfileInjector {
+ Function &F;
+ FunctionAnalysisManager &FAM;
+
+public:
+ static bool supportsBranchWeights(const Instruction &I) {
+ return isa<BranchInst>(&I) ||
+
+ isa<SwitchInst>(&I) ||
+
+ isa<IndirectBrInst>(&I) || isa<SelectInst>(&I) ||
+ isa<CallBrInst>(&I);
+ }
+
+ ProfileInjector(Function &F, FunctionAnalysisManager &FAM) : F(F), FAM(FAM) {}
+ bool inject();
+};
+} // namespace
+
+bool ProfileInjector::inject() {
+ auto &BPI = FAM.getResult<BranchProbabilityAnalysis>(F);
+
+ bool Changed = false;
+ for (auto &BB : F) {
+ if (succ_size(&BB) <= 1)
+ continue;
+ auto *Term = BB.getTerminator();
+ assert(Term);
+ if (Term->getMetadata(LLVMContext::MD_prof) ||
+ !supportsBranchWeights(*Term))
+ continue;
+ SmallVector<BranchProbability> Probs;
+ Probs.reserve(Term->getNumSuccessors());
+ for (auto I = 0U, E = Term->getNumSuccessors(); I < E; ++I)
+ Probs.emplace_back(BPI.getEdgeProbability(&BB, Term->getSuccessor(I)));
+
+ const auto *FirstZeroDenominator =
+ find_if(Probs, [](const BranchProbability &P) {
+ return P.getDenominator() == 0;
+ });
+ (void)FirstZeroDenominator;
+ assert(FirstZeroDenominator == Probs.end());
+ const auto *FirstNonzeroNumerator =
+ find_if(Probs, [](const BranchProbability &P) {
+ return P.getNumerator() != 0;
+ });
+ assert(FirstNonzeroNumerator != Probs.end());
+ DynamicAPInt LCM(Probs[0].getDenominator());
+ DynamicAPInt GCD(FirstNonzeroNumerator->getNumerator());
+ for (const auto &Prob : drop_begin(Probs)) {
+ if (!Prob.getNumerator())
+ continue;
+ LCM = llvm::lcm(LCM, DynamicAPInt(Prob.getDenominator()));
+ GCD = llvm::lcm(GCD, DynamicAPInt(Prob.getNumerator()));
+ }
+ SmallVector<uint32_t> Weights;
+ Weights.reserve(Term->getNumSuccessors());
+ for (const auto &Prob : Probs) {
+ auto W = Prob.getNumerator() * LCM / GCD;
+ Weights.emplace_back(static_cast<int32_t>((int64_t)W));
+ }
+ setBranchWeights(*Term, Weights, false);
+ Changed = true;
+ }
+ return Changed;
+}
+
+PreservedAnalyses ProfileInjectorPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ ProfileInjector PI(F, FAM);
+ if (!PI.inject())
+ return PreservedAnalyses::all();
+
+ return PreservedAnalyses::none();
+}
+
+PreservedAnalyses ProfileVerifierPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ for (auto &BB : F)
+ if (succ_size(&BB) >= 2)
+ if (auto *Term = BB.getTerminator())
+ if (ProfileInjector::supportsBranchWeights(*Term))
+ if (!Term->getMetadata(LLVMContext::MD_prof))
+ F.getContext().emitError("Profile verification failed");
+
+ return PreservedAnalyses::none();
+}
diff --git a/llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll b/llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll
new file mode 100644
index 0000000000000..fda6a8f92d196
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll
@@ -0,0 +1,20 @@
+; Test that prof-inject only injects missing metadata
+
+; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s
+
+define void @foo(i32 %i) {
+ %c = icmp eq i32 %i, 0
+ br i1 %c, label %yes, label %no, !prof !0
+yes:
+ br i1 %c, label %yes2, label %no
+yes2:
+ ret void
+no:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: br i1 %c, label %yes, label %no, !prof !0
+; CHECK: br i1 %c, label %yes2, label %no, !prof !1
+; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: !1 = !{!"branch_weights", i32 429496729, i32 715827882}
diff --git a/llvm/test/Transforms/PGOProfile/prof-verify-existing.ll b/llvm/test/Transforms/PGOProfile/prof-verify-existing.ll
new file mode 100644
index 0000000000000..ea4f0f9f1dadf
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/prof-verify-existing.ll
@@ -0,0 +1,21 @@
+; Test that prof-inject does not modify existing metadata (incl. "unknown")
+
+; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s
+; RUN: opt -passes=prof-verify %s -S --disable-output
+
+define void @foo(i32 %i) {
+ %c = icmp eq i32 %i, 0
+ br i1 %c, label %yes, label %no, !prof !0
+yes:
+ br i1 %c, label %yes2, label %no, !prof !1
+yes2:
+ ret void
+no:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 2}
+!1 = !{!"unknown"}
+; CHECK: br i1 %c, label %yes, label %no, !prof !0
+; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: !1 = !{!"unknown"}
diff --git a/llvm/test/Transforms/PGOProfile/prof-verify.ll b/llvm/test/Transforms/PGOProfile/prof-verify.ll
new file mode 100644
index 0000000000000..f4d2b2e51dc59
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/prof-verify.ll
@@ -0,0 +1,19 @@
+; Test prof-inject and prof-verify
+
+; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s --check-prefix=INJECT
+; RUN: not opt -passes=prof-verify %s -S -o - 2>&1 | FileCheck %s --check-prefix=VERIFY
+; RUN: opt -passes=prof-inject,prof-verify %s --disable-output
+
+define void @foo(i32 %i) {
+ %c = icmp eq i32 %i, 0
+ br i1 %c, label %yes, label %no
+yes:
+ ret void
+no:
+ ret void
+}
+
+; INJECT: br i1 %c, label %yes, label %no, !prof !0
+; INJECT: !0 = !{!"branch_weights", i32 429496729, i32 715827882}
+
+; VERIFY: Profile verification failed
\ No newline at end of file
More information about the llvm-commits
mailing list