[llvm] df2d2d1 - [PGO] Add ProfileInjector and ProfileVerifier passes (#147388)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 23 12:35:02 PDT 2025
Author: Mircea Trofin
Date: 2025-07-23T21:34:58+02:00
New Revision: df2d2d125beffb76d5c526a5661fbb11cbcff83b
URL: https://github.com/llvm/llvm-project/commit/df2d2d125beffb76d5c526a5661fbb11cbcff83b
DIFF: https://github.com/llvm/llvm-project/commit/df2d2d125beffb76d5c526a5661fbb11cbcff83b.diff
LOG: [PGO] Add ProfileInjector and ProfileVerifier passes (#147388)
Adding 2 passes, one to inject `MD_prof` and one to check its presence. A subsequent patch will add these (similar to debugify) to `opt` (and, eventually, a variant of this, to `llc`)
Tracking issue: #147390
Added:
llvm/include/llvm/Transforms/Utils/ProfileVerify.h
llvm/lib/Transforms/Utils/ProfileVerify.cpp
llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll
llvm/test/Transforms/PGOProfile/prof-verify-existing.ll
llvm/test/Transforms/PGOProfile/prof-verify.ll
Modified:
llvm/lib/Passes/PassBuilder.cpp
llvm/lib/Passes/PassRegistry.def
llvm/lib/Transforms/Utils/CMakeLists.txt
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Utils/ProfileVerify.h b/llvm/include/llvm/Transforms/Utils/ProfileVerify.h
new file mode 100644
index 0000000000000..7834305b569d2
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/ProfileVerify.h
@@ -0,0 +1,36 @@
+//===- ProfileVerify.h - Verify profile info for testing ----------*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Inject profile information, as part of tests, to verify passes don't
+// accidentally drop it.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_UTILS_PROFILEVERIFY_H
+#define LLVM_TRANSFORMS_UTILS_PROFILEVERIFY_H
+
+#include "llvm/IR/Analysis.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+/// Inject MD_prof metadata where it's missing. Used for testing that passes
+/// don't accidentally drop this metadata.
+class ProfileInjectorPass : public PassInfoMixin<ProfileInjectorPass> {
+public:
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+/// Checks that MD_prof is present on every instruction that supports it. Used
+/// in conjunction with the ProfileInjectorPass. MD_prof "unknown" is considered
+/// valid (i.e. !{!"unknown"})
+class ProfileVerifierPass : public PassInfoMixin<ProfileVerifierPass> {
+public:
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+} // namespace llvm
+#endif
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index cff7ab52f1536..f810368a84940 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -364,6 +364,7 @@
#include "llvm/Transforms/Utils/MoveAutoInit.h"
#include "llvm/Transforms/Utils/NameAnonGlobals.h"
#include "llvm/Transforms/Utils/PredicateInfo.h"
+#include "llvm/Transforms/Utils/ProfileVerify.h"
#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
#include "llvm/Transforms/Utils/StripGCRelocates.h"
#include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index caa78b613b901..bb7ccdb2bc187 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -520,6 +520,8 @@ FUNCTION_PASS("print<regions>", RegionInfoPrinterPass(errs()))
FUNCTION_PASS("print<scalar-evolution>", ScalarEvolutionPrinterPass(errs()))
FUNCTION_PASS("print<stack-safety-local>", StackSafetyPrinterPass(errs()))
FUNCTION_PASS("print<uniformity>", UniformityInfoPrinterPass(errs()))
+FUNCTION_PASS("prof-inject", ProfileInjectorPass())
+FUNCTION_PASS("prof-verify", ProfileVerifierPass())
FUNCTION_PASS("reassociate", ReassociatePass())
FUNCTION_PASS("redundant-dbg-inst-elim", RedundantDbgInstEliminationPass())
FUNCTION_PASS("replace-with-veclib", ReplaceWithVeclib())
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index f7e66eca6aeb3..a4fa0e2e6970d 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -68,6 +68,7 @@ add_llvm_component_library(LLVMTransformUtils
MoveAutoInit.cpp
NameAnonGlobals.cpp
PredicateInfo.cpp
+ ProfileVerify.cpp
PromoteMemoryToRegister.cpp
RelLookupTableConverter.cpp
ScalarEvolutionExpander.cpp
diff --git a/llvm/lib/Transforms/Utils/ProfileVerify.cpp b/llvm/lib/Transforms/Utils/ProfileVerify.cpp
new file mode 100644
index 0000000000000..b972132eb8c42
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/ProfileVerify.cpp
@@ -0,0 +1,129 @@
+//===- ProfileVerify.cpp - Verify profile info for testing ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/ProfileVerify.h"
+#include "llvm/ADT/DynamicAPInt.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Analysis.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/ProfDataUtils.h"
+#include "llvm/Support/BranchProbability.h"
+
+using namespace llvm;
+namespace {
+class ProfileInjector {
+ Function &F;
+ FunctionAnalysisManager &FAM;
+
+public:
+ static const Instruction *
+ getTerminatorBenefitingFromMDProf(const BasicBlock &BB) {
+ if (succ_size(&BB) < 2)
+ return nullptr;
+ auto *Term = BB.getTerminator();
+ return (isa<BranchInst>(Term) || isa<SwitchInst>(Term) ||
+ isa<IndirectBrInst>(Term) || isa<CallBrInst>(Term))
+ ? Term
+ : nullptr;
+ }
+
+ static Instruction *getTerminatorBenefitingFromMDProf(BasicBlock &BB) {
+ return const_cast<Instruction *>(
+ getTerminatorBenefitingFromMDProf(const_cast<const BasicBlock &>(BB)));
+ }
+
+ ProfileInjector(Function &F, FunctionAnalysisManager &FAM) : F(F), FAM(FAM) {}
+ bool inject();
+};
+} // namespace
+
+// FIXME: currently this injects only for terminators. Select isn't yet
+// supported.
+bool ProfileInjector::inject() {
+ // Get whatever branch probability info can be derived from the given IR -
+ // whether it has or not metadata. The main intention for this pass is to
+ // ensure that other passes don't drop or "forget" to update MD_prof. We do
+ // this as a mode in which lit tests would run. We want to avoid changing the
+ // behavior of those tests. A pass may use BPI (or BFI, which is computed from
+ // BPI). If no metadata is present, BPI is guesstimated by
+ // BranchProbabilityAnalysis. The injector (this pass) only persists whatever
+ // information the analysis provides, in other words, the pass being tested
+ // will get the same BPI it does if the injector wasn't running.
+ auto &BPI = FAM.getResult<BranchProbabilityAnalysis>(F);
+
+ bool Changed = false;
+ for (auto &BB : F) {
+ auto *Term = getTerminatorBenefitingFromMDProf(BB);
+ if (!Term || Term->getMetadata(LLVMContext::MD_prof))
+ continue;
+ SmallVector<BranchProbability> Probs;
+ Probs.reserve(Term->getNumSuccessors());
+ for (auto I = 0U, E = Term->getNumSuccessors(); I < E; ++I)
+ Probs.emplace_back(BPI.getEdgeProbability(&BB, Term->getSuccessor(I)));
+
+ assert(llvm::find_if(Probs,
+ [](const BranchProbability &P) {
+ return P.isUnknown();
+ }) == Probs.end() &&
+ "All branch probabilities should be valid");
+ const auto *FirstZeroDenominator =
+ find_if(Probs, [](const BranchProbability &P) {
+ return P.getDenominator() == 0;
+ });
+ (void)FirstZeroDenominator;
+ assert(FirstZeroDenominator == Probs.end());
+ const auto *FirstNonZeroNumerator =
+ find_if(Probs, [](const BranchProbability &P) { return !P.isZero(); });
+ assert(FirstNonZeroNumerator != Probs.end());
+ DynamicAPInt LCM(Probs[0].getDenominator());
+ DynamicAPInt GCD(FirstNonZeroNumerator->getNumerator());
+ for (const auto &Prob : drop_begin(Probs)) {
+ if (!Prob.getNumerator())
+ continue;
+ LCM = llvm::lcm(LCM, DynamicAPInt(Prob.getDenominator()));
+ GCD = llvm::gcd(GCD, DynamicAPInt(Prob.getNumerator()));
+ }
+ SmallVector<uint32_t> Weights;
+ Weights.reserve(Term->getNumSuccessors());
+ for (const auto &Prob : Probs) {
+ DynamicAPInt W =
+ (Prob.getNumerator() * LCM / GCD) / Prob.getDenominator();
+ Weights.emplace_back(static_cast<uint32_t>((int64_t)W));
+ }
+ setBranchWeights(*Term, Weights, /*IsExpected=*/false);
+ Changed = true;
+ }
+ return Changed;
+}
+
+PreservedAnalyses ProfileInjectorPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ ProfileInjector PI(F, FAM);
+ if (!PI.inject())
+ return PreservedAnalyses::all();
+
+ return PreservedAnalyses::none();
+}
+
+PreservedAnalyses ProfileVerifierPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ for (const auto &BB : F)
+ if (const auto *Term =
+ ProfileInjector::getTerminatorBenefitingFromMDProf(BB))
+ if (!Term->getMetadata(LLVMContext::MD_prof))
+ F.getContext().emitError("Profile verification failed");
+
+ return PreservedAnalyses::none();
+}
diff --git a/llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll b/llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll
new file mode 100644
index 0000000000000..07e1f2d3c6127
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll
@@ -0,0 +1,20 @@
+; Test that prof-inject only injects missing metadata
+
+; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s
+
+define void @foo(i32 %i) {
+ %c = icmp eq i32 %i, 0
+ br i1 %c, label %yes, label %no, !prof !0
+yes:
+ br i1 %c, label %yes2, label %no
+yes2:
+ ret void
+no:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: br i1 %c, label %yes, label %no, !prof !0
+; CHECK: br i1 %c, label %yes2, label %no, !prof !1
+; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: !1 = !{!"branch_weights", i32 3, i32 5}
diff --git a/llvm/test/Transforms/PGOProfile/prof-verify-existing.ll b/llvm/test/Transforms/PGOProfile/prof-verify-existing.ll
new file mode 100644
index 0000000000000..ea4f0f9f1dadf
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/prof-verify-existing.ll
@@ -0,0 +1,21 @@
+; Test that prof-inject does not modify existing metadata (incl. "unknown")
+
+; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s
+; RUN: opt -passes=prof-verify %s -S --disable-output
+
+define void @foo(i32 %i) {
+ %c = icmp eq i32 %i, 0
+ br i1 %c, label %yes, label %no, !prof !0
+yes:
+ br i1 %c, label %yes2, label %no, !prof !1
+yes2:
+ ret void
+no:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 2}
+!1 = !{!"unknown"}
+; CHECK: br i1 %c, label %yes, label %no, !prof !0
+; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: !1 = !{!"unknown"}
diff --git a/llvm/test/Transforms/PGOProfile/prof-verify.ll b/llvm/test/Transforms/PGOProfile/prof-verify.ll
new file mode 100644
index 0000000000000..c83475ab5d18e
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/prof-verify.ll
@@ -0,0 +1,19 @@
+; Test prof-inject and prof-verify
+
+; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s --check-prefix=INJECT
+; RUN: not opt -passes=prof-verify %s -S -o - 2>&1 | FileCheck %s --check-prefix=VERIFY
+; RUN: opt -passes=prof-inject,prof-verify %s --disable-output
+
+define void @foo(i32 %i) {
+ %c = icmp eq i32 %i, 0
+ br i1 %c, label %yes, label %no
+yes:
+ ret void
+no:
+ ret void
+}
+
+; INJECT: br i1 %c, label %yes, label %no, !prof !0
+; INJECT: !0 = !{!"branch_weights", i32 3, i32 5}
+
+; VERIFY: Profile verification failed
\ No newline at end of file
More information about the llvm-commits
mailing list