[llvm] [llvm][ctx_profile] Add instrumentation (PR #90136)
Mircea Trofin via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 25 15:25:21 PDT 2024
https://github.com/mtrofin created https://github.com/llvm/llvm-project/pull/90136
This adds instrumenting callsites to PGOInstrumentation, *if* contextual profiling is requested. The latter also enables inserting counters in the entry basic block and disables value profiling (the latter is a point in time change)
This change adds the skeleton of the contextual profiling lowering pass, just so we can introduce the flag controlling that and the API to check that. The actual lowering pass will be introduced in a subsequent patch.
(Tracking Issue: #89287, RFC referenced there)
>From e4d731bc3ba37ea3e294992ca7b8475197d3ec64 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin at google.com>
Date: Wed, 24 Apr 2024 07:51:55 -0700
Subject: [PATCH] [llvm][ctx_profile] Add instrumentation
This adds instrumenting callsites to PGOInstrumentation, *if* contextual
profiling is requested. The latter also enables inserting counters in
the entry basic block and disables value profiling (the latter is a
point in time change)
This change adds the skeleton of the contextual profiling lowering pass,
just so we can introduce the flag controlling that and the API to check
that. The actual lowering pass will be introduced in a subsequent patch.
(Tracking Issue: #89287, RFC referenced there)
---
.../Instrumentation/PGOCtxProfLowering.h | 24 ++++++++
.../Transforms/Instrumentation/CMakeLists.txt | 1 +
.../Instrumentation/PGOCtxProfLowering.cpp | 19 ++++++
.../Instrumentation/PGOInstrumentation.cpp | 51 +++++++++++++---
.../PGOProfile/ctx-instrumentation.ll | 61 +++++++++++++++++++
5 files changed, 149 insertions(+), 7 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h
create mode 100644 llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
create mode 100644 llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h
new file mode 100644
index 00000000000000..ec71d770fd6ef2
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h
@@ -0,0 +1,24 @@
+//===-- PGOCtxProfLowering.h - Contextual PGO Instr. Lowering ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the PGOCtxProfLoweringPass class.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_PGOCTXPROFLOWERING_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_PGOCTXPROFLOWERING_H
+
+namespace llvm {
+class Type;
+
+class PGOCtxProfLoweringPass {
+public:
+ explicit PGOCtxProfLoweringPass() = default;
+ static bool isContextualIRPGOEnabled();
+};
+} // namespace llvm
+#endif
\ No newline at end of file
diff --git a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
index 981405329389f4..8d345d394b51a2 100644
--- a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
+++ b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
@@ -14,6 +14,7 @@ add_llvm_component_library(LLVMInstrumentation
InstrProfiling.cpp
KCFI.cpp
LowerAllowCheckPass.cpp
+ PGOCtxProfLowering.cpp
PGOForceFunctionAttrs.cpp
PGOInstrumentation.cpp
PGOMemOPSizeOpt.cpp
diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
new file mode 100644
index 00000000000000..3cf091c061541b
--- /dev/null
+++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
@@ -0,0 +1,19 @@
+//===- PGOCtxProfLowering.cpp - Contextual PGO Instr. Lowering ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+
+#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace llvm;
+
+static cl::list<std::string> ContextRoots("profile-context-root");
+
+bool PGOCtxProfLoweringPass::isContextualIRPGOEnabled() {
+ return !ContextRoots.empty();
+}
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index a7b7556685e443..c3d39075b72f3e 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -110,6 +110,7 @@
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Instrumentation/BlockCoverageInference.h"
#include "llvm/Transforms/Instrumentation/CFGMST.h"
+#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/MisExpect.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -333,6 +334,16 @@ extern cl::opt<bool> EnableVTableValueProfiling;
extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
} // namespace llvm
+bool shouldInstrumentEntryBB() {
+ return PGOInstrumentEntry ||
+ PGOCtxProfLoweringPass::isContextualIRPGOEnabled();
+}
+
+bool isValueProfilingDisabled() {
+ return DisableValueProfiling ||
+ PGOCtxProfLoweringPass::isContextualIRPGOEnabled();
+}
+
// Return a string describing the branch condition that can be
// used in static branch probability heuristics:
static std::string getBranchCondString(Instruction *TI) {
@@ -379,7 +390,7 @@ static GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS) {
uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
if (IsCS)
ProfileVersion |= VARIANT_MASK_CSIR_PROF;
- if (PGOInstrumentEntry)
+ if (shouldInstrumentEntryBB())
ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
@@ -861,7 +872,7 @@ static void instrumentOneFunc(
}
FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
- F, TLI, ComdatMembers, true, BPI, BFI, IsCS, PGOInstrumentEntry,
+ F, TLI, ComdatMembers, true, BPI, BFI, IsCS, shouldInstrumentEntryBB(),
PGOBlockCoverage);
auto Name = FuncInfo.FuncNameVar;
@@ -883,6 +894,33 @@ static void instrumentOneFunc(
unsigned NumCounters =
InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
+ if (PGOCtxProfLoweringPass::isContextualIRPGOEnabled()) {
+ auto *CSIntrinsic =
+ Intrinsic::getDeclaration(M, Intrinsic::instrprof_callsite);
+ auto Visit = [&](llvm::function_ref<void(CallBase * CB)> Visitor) {
+ for (auto &BB : F)
+ for (auto &Instr : BB)
+ if (auto *CS = dyn_cast<CallBase>(&Instr)) {
+ if ((CS->getCalledFunction() &&
+ CS->getCalledFunction()->isIntrinsic()) ||
+ dyn_cast<InlineAsm>(CS->getCalledOperand()))
+ continue;
+ Visitor(CS);
+ }
+ };
+ uint32_t TotalNrCallsites = 0;
+ Visit([&TotalNrCallsites](auto *) { ++TotalNrCallsites; });
+ uint32_t CallsiteIndex = 0;
+
+ Visit([&](auto *CB) {
+ IRBuilder<> Builder(CB);
+ Builder.CreateCall(CSIntrinsic,
+ {Name, CFGHash, Builder.getInt32(TotalNrCallsites),
+ Builder.getInt32(CallsiteIndex++),
+ CB->getCalledOperand()});
+ });
+ }
+
uint32_t I = 0;
if (PGOTemporalInstrumentation) {
NumCounters += PGOBlockCoverage ? 8 : 1;
@@ -914,7 +952,7 @@ static void instrumentOneFunc(
FuncInfo.FunctionHash);
assert(I == NumCounters);
- if (DisableValueProfiling)
+ if (isValueProfilingDisabled())
return;
NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size();
@@ -1676,7 +1714,7 @@ void SelectInstVisitor::visitSelectInst(SelectInst &SI) {
// Traverse all valuesites and annotate the instructions for all value kind.
void PGOUseFunc::annotateValueSites() {
- if (DisableValueProfiling)
+ if (isValueProfilingDisabled())
return;
// Create the PGOFuncName meta data.
@@ -2015,9 +2053,8 @@ static bool annotateAllFunctions(
// If the profile marked as always instrument the entry BB, do the
// same. Note this can be overwritten by the internal option in CFGMST.h
- bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
- if (PGOInstrumentEntry.getNumOccurrences() > 0)
- InstrumentFuncEntry = PGOInstrumentEntry;
+ bool InstrumentFuncEntry =
+ PGOReader->instrEntryBBEnabled() || shouldInstrumentEntryBB();
bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
for (auto &F : M) {
if (skipPGOUse(F))
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
new file mode 100644
index 00000000000000..cba2285b888ee1
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 4
+; RUN: opt -passes=pgo-instr-gen -profile-context-root=an_entrypoint \
+; RUN: -S < %s | FileCheck --check-prefix=INSTRUMENT %s
+
+declare void @bar()
+
+;.
+; INSTRUMENT: @__llvm_profile_raw_version = hidden constant i64 360287970189639690, comdat
+; INSTRUMENT: @__profn_foo = private constant [3 x i8] c"foo"
+; INSTRUMENT: @__profn_an_entrypoint = private constant [13 x i8] c"an_entrypoint"
+;.
+define void @foo(i32 %a) {
+; INSTRUMENT-LABEL: define void @foo(
+; INSTRUMENT-SAME: i32 [[A:%.*]]) {
+; INSTRUMENT-NEXT: call void @llvm.instrprof.increment(ptr @__profn_foo, i64 146835647075900052, i32 2, i32 0)
+; INSTRUMENT-NEXT: [[T:%.*]] = icmp eq i32 [[A]], 0
+; INSTRUMENT-NEXT: br i1 [[T]], label [[YES:%.*]], label [[NO:%.*]]
+; INSTRUMENT: yes:
+; INSTRUMENT-NEXT: call void @llvm.instrprof.increment(ptr @__profn_foo, i64 146835647075900052, i32 2, i32 1)
+; INSTRUMENT-NEXT: br label [[EXIT:%.*]]
+; INSTRUMENT: no:
+; INSTRUMENT-NEXT: call void @llvm.instrprof.callsite(ptr @__profn_foo, i64 146835647075900052, i32 1, i32 0, ptr @bar)
+; INSTRUMENT-NEXT: call void @bar()
+; INSTRUMENT-NEXT: br label [[EXIT]]
+; INSTRUMENT: exit:
+; INSTRUMENT-NEXT: ret void
+;
+ %t = icmp eq i32 %a, 0
+ br i1 %t, label %yes, label %no
+yes:
+ br label %exit
+no:
+ call void @bar()
+ br label %exit
+exit:
+ ret void
+}
+
+define void @an_entrypoint(i32 %a) {
+; INSTRUMENT-LABEL: define void @an_entrypoint(
+; INSTRUMENT-SAME: i32 [[A:%.*]]) {
+; INSTRUMENT-NEXT: call void @llvm.instrprof.increment(ptr @__profn_an_entrypoint, i64 784007058953177093, i32 2, i32 0)
+; INSTRUMENT-NEXT: [[T:%.*]] = icmp eq i32 [[A]], 0
+; INSTRUMENT-NEXT: br i1 [[T]], label [[YES:%.*]], label [[NO:%.*]]
+; INSTRUMENT: yes:
+; INSTRUMENT-NEXT: call void @llvm.instrprof.increment(ptr @__profn_an_entrypoint, i64 784007058953177093, i32 2, i32 1)
+; INSTRUMENT-NEXT: ret void
+; INSTRUMENT: no:
+; INSTRUMENT-NEXT: ret void
+;
+ %t = icmp eq i32 %a, 0
+ br i1 %t, label %yes, label %no
+
+yes:
+ ret void
+no:
+ ret void
+}
+;.
+; INSTRUMENT: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+;.
More information about the llvm-commits
mailing list