[llvm] 47b0758 - [SampleFDO] Persist profile staleness metrics into binary
via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 9 22:35:15 PST 2022
Author: wlei
Date: 2022-11-09T22:34:33-08:00
New Revision: 47b0758049eab549c51e77c8f8b5560498af5b80
URL: https://github.com/llvm/llvm-project/commit/47b0758049eab549c51e77c8f8b5560498af5b80
DIFF: https://github.com/llvm/llvm-project/commit/47b0758049eab549c51e77c8f8b5560498af5b80.diff
LOG: [SampleFDO] Persist profile staleness metrics into binary
With https://reviews.llvm.org/D136627, now we have the metrics for profile staleness based on profile statistics, monitoring the profile staleness in real-time can help user quickly identify performance issues. For a production scenario, the build is usually incremental and if we want the real-time metrics, we should store/cache all the old object's metrics somewhere and pull them in a post-build time. To make it more convenient, this patch add an option to persist them into the object binary, the metrics can be reported right away by decoding the binary rather than polling the previous stdout/stderrs from a cache system.
For implementation, it writes the statistics first into a new metadata section(llvm.stats) then encode into a special ELF `.llvm_stats` section. The section data is formatted as a list of key/value pair so that future statistics can be easily extended. This is also under a new switch(`-persist-profile-staleness`)
In terms of size overhead, the metrics are computed at module level, so the size overhead should be small, measured on one of our internal service, it costs less than < 1MB for a 10GB+ binary.
Reviewed By: wenlei
Differential Revision: https://reviews.llvm.org/D136698
Added:
Modified:
llvm/include/llvm/IR/MDBuilder.h
llvm/include/llvm/MC/MCObjectFileInfo.h
llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
llvm/lib/IR/MDBuilder.cpp
llvm/lib/MC/MCObjectFileInfo.cpp
llvm/lib/Transforms/IPO/SampleProfile.cpp
llvm/test/Transforms/SampleProfile/profile-mismatch.ll
llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h
index f31a769b6b442..bd542bd0d2b2b 100644
--- a/llvm/include/llvm/IR/MDBuilder.h
+++ b/llvm/include/llvm/IR/MDBuilder.h
@@ -80,6 +80,10 @@ class MDBuilder {
/// Return metadata containing the pseudo probe descriptor for a function.
MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, Function *F);
+ /// Return metadata containing llvm statistics.
+ MDNode *
+ createLLVMStats(ArrayRef<std::pair<StringRef, uint64_t>> LLVMStatsVec);
+
//===------------------------------------------------------------------===//
// Range metadata.
//===------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h
index 847b9ffc3cfc0..d743bc5371b58 100644
--- a/llvm/include/llvm/MC/MCObjectFileInfo.h
+++ b/llvm/include/llvm/MC/MCObjectFileInfo.h
@@ -180,6 +180,9 @@ class MCObjectFileInfo {
MCSection *PseudoProbeSection = nullptr;
MCSection *PseudoProbeDescSection = nullptr;
+ // Section for metadata of llvm statistics.
+ MCSection *LLVMStatsSection = nullptr;
+
// ELF specific sections.
MCSection *DataRelROSection = nullptr;
MCSection *MergeableConst4Section = nullptr;
@@ -366,6 +369,8 @@ class MCObjectFileInfo {
MCSection *getPseudoProbeDescSection(StringRef FuncName) const;
+ MCSection *getLLVMStatsSection() const;
+
MCSection *getPCSection(StringRef Name, const MCSection *TextSec) const;
// ELF specific sections.
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 01f42477ad196..89148350c49aa 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -58,6 +58,7 @@
#include "llvm/MC/MCValue.h"
#include "llvm/MC/SectionKind.h"
#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/Base64.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/ErrorHandling.h"
@@ -366,6 +367,31 @@ void TargetLoweringObjectFileELF::emitModuleMetadata(MCStreamer &Streamer,
}
}
+ if (NamedMDNode *LLVMStats = M.getNamedMetadata("llvm.stats")) {
+ // Emit the metadata for llvm statistics into .llvm_stats section, which is
+ // formatted as a list of key/value pair, the value is base64 encoded.
+ auto *S = C.getObjectFileInfo()->getLLVMStatsSection();
+ Streamer.switchSection(S);
+ for (const auto *Operand : LLVMStats->operands()) {
+ const auto *MD = cast<MDNode>(Operand);
+ assert(MD->getNumOperands() % 2 == 0 &&
+ ("Operand num should be even for a list of key/value pair"));
+ for (size_t I = 0; I < MD->getNumOperands(); I += 2) {
+ // Encode the key string size.
+ auto *Key = cast<MDString>(MD->getOperand(I));
+ Streamer.emitULEB128IntValue(Key->getString().size());
+ Streamer.emitBytes(Key->getString());
+ // Encode the value into a Base64 string.
+ std::string Value = encodeBase64(
+ Twine(mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1))
+ ->getZExtValue())
+ .str());
+ Streamer.emitULEB128IntValue(Value.size());
+ Streamer.emitBytes(Value);
+ }
+ }
+ }
+
unsigned Version = 0;
unsigned Flags = 0;
StringRef Section;
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index d028eb4a21dd9..df08eb2063efb 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -344,3 +344,15 @@ MDNode *MDBuilder::createPseudoProbeDesc(uint64_t GUID, uint64_t Hash,
Ops[2] = createString(F->getName());
return MDNode::get(Context, Ops);
}
+
+MDNode *
+MDBuilder::createLLVMStats(ArrayRef<std::pair<StringRef, uint64_t>> LLVMStats) {
+ auto *Int64Ty = Type::getInt64Ty(Context);
+ SmallVector<Metadata *, 4> Ops(LLVMStats.size() * 2);
+ for (size_t I = 0; I < LLVMStats.size(); I++) {
+ Ops[I * 2] = createString(LLVMStats[I].first);
+ Ops[I * 2 + 1] =
+ createConstant(ConstantInt::get(Int64Ty, LLVMStats[I].second));
+ }
+ return MDNode::get(Context, Ops);
+}
diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp
index b5460ced752a6..57839473205f9 100644
--- a/llvm/lib/MC/MCObjectFileInfo.cpp
+++ b/llvm/lib/MC/MCObjectFileInfo.cpp
@@ -531,6 +531,8 @@ void MCObjectFileInfo::initELFMCObjectFileInfo(const Triple &T, bool Large) {
PseudoProbeSection = Ctx->getELFSection(".pseudo_probe", DebugSecType, 0);
PseudoProbeDescSection =
Ctx->getELFSection(".pseudo_probe_desc", DebugSecType, 0);
+
+ LLVMStatsSection = Ctx->getELFSection(".llvm_stats", ELF::SHT_PROGBITS, 0);
}
void MCObjectFileInfo::initGOFFMCObjectFileInfo(const Triple &T) {
@@ -1199,6 +1201,10 @@ MCObjectFileInfo::getPseudoProbeDescSection(StringRef FuncName) const {
return PseudoProbeDescSection;
}
+MCSection *MCObjectFileInfo::getLLVMStatsSection() const {
+ return LLVMStatsSection;
+}
+
MCSection *MCObjectFileInfo::getPCSection(StringRef Name,
const MCSection *TextSec) const {
if (Ctx->getObjectFileType() != MCContext::IsELF)
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index f01a3ef867619..66c1e5b4bd594 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -133,6 +133,11 @@ static cl::opt<bool> ReportProfileStaleness(
"report-profile-staleness", cl::Hidden, cl::init(false),
cl::desc("Compute and report stale profile statistical metrics."));
+static cl::opt<bool> PersistProfileStaleness(
+ "persist-profile-staleness", cl::Hidden, cl::init(false),
+ cl::desc("Compute stale profile statistical metrics and write it into the "
+ "native object file(.llvm_stats section)."));
+
static cl::opt<bool> ProfileSampleAccurate(
"profile-sample-accurate", cl::Hidden, cl::init(false),
cl::desc("If the sample profile is accurate, we will mark all un-sampled "
@@ -2041,7 +2046,7 @@ bool SampleProfileLoader::doInitialization(Module &M,
}
}
- if (ReportProfileStaleness) {
+ if (ReportProfileStaleness || PersistProfileStaleness) {
MatchingManager =
std::make_unique<SampleProfileMatcher>(M, *Reader, ProbeManager.get());
}
@@ -2150,18 +2155,42 @@ void SampleProfileMatcher::detectProfileMismatch() {
detectProfileMismatch(F, *FS);
}
- if (FunctionSamples::ProfileIsProbeBased) {
- errs() << "(" << NumMismatchedFuncHash << "/" << TotalProfiledFunc << ")"
- << " of functions' profile are invalid and "
- << " (" << MismatchedFuncHashSamples << "/" << TotalFuncHashSamples
+ if (ReportProfileStaleness) {
+ if (FunctionSamples::ProfileIsProbeBased) {
+ errs() << "(" << NumMismatchedFuncHash << "/" << TotalProfiledFunc << ")"
+ << " of functions' profile are invalid and "
+ << " (" << MismatchedFuncHashSamples << "/" << TotalFuncHashSamples
+ << ")"
+ << " of samples are discarded due to function hash mismatch.\n";
+ }
+ errs() << "(" << NumMismatchedCallsite << "/" << TotalProfiledCallsite
<< ")"
- << " of samples are discarded due to function hash mismatch.\n";
+ << " of callsites' profile are invalid and "
+ << "(" << MismatchedCallsiteSamples << "/" << TotalCallsiteSamples
+ << ")"
+ << " of samples are discarded due to callsite location mismatch.\n";
+ }
+
+ if (PersistProfileStaleness) {
+ LLVMContext &Ctx = M.getContext();
+ MDBuilder MDB(Ctx);
+
+ SmallVector<std::pair<StringRef, uint64_t>> ProfStatsVec;
+ if (FunctionSamples::ProfileIsProbeBased) {
+ ProfStatsVec.emplace_back("NumMismatchedFuncHash", NumMismatchedFuncHash);
+ ProfStatsVec.emplace_back("TotalProfiledFunc", TotalProfiledFunc);
+ ProfStatsVec.emplace_back("MismatchedFuncHashSamples",
+ MismatchedFuncHashSamples);
+ ProfStatsVec.emplace_back("TotalFuncHashSamples", TotalFuncHashSamples);
+ }
+ ProfStatsVec.emplace_back("MismatchedCallsiteSamples",
+ MismatchedCallsiteSamples);
+ ProfStatsVec.emplace_back("TotalCallsiteSamples", TotalCallsiteSamples);
+
+ auto *MD = MDB.createLLVMStats(ProfStatsVec);
+ auto *NMD = M.getOrInsertNamedMetadata("llvm.stats");
+ NMD->addOperand(MD);
}
- errs() << "(" << NumMismatchedCallsite << "/" << TotalProfiledCallsite << ")"
- << " of callsites' profile are invalid and "
- << "(" << MismatchedCallsiteSamples << "/" << TotalCallsiteSamples
- << ")"
- << " of samples are discarded due to callsite location mismatch.\n";
}
bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
@@ -2208,7 +2237,7 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
assert(SymbolMap.count(StringRef()) == 0 &&
"No empty StringRef should be added in SymbolMap");
- if (ReportProfileStaleness)
+ if (ReportProfileStaleness || PersistProfileStaleness)
MatchingManager->detectProfileMismatch();
bool retval = false;
diff --git a/llvm/test/Transforms/SampleProfile/profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/profile-mismatch.ll
index 68f8e19b10701..413c7120dcf6e 100644
--- a/llvm/test/Transforms/SampleProfile/profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-mismatch.ll
@@ -1,8 +1,15 @@
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-mismatch.prof -report-profile-staleness -S 2>%t
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-mismatch.prof -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll
; RUN: FileCheck %s --input-file %t
+; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD
+; RUN: llc < %t.ll -filetype=obj -o %t.obj
+; RUN: llvm-objdump --section-headers %t.obj | FileCheck %s --check-prefix=CHECK-OBJ
; CHECK: (2/3) of callsites' profile are invalid and (20/30) of samples are discarded due to callsite location mismatch.
+; CHECK-MD: ![[#]] = !{!"MismatchedCallsiteSamples", i64 20, !"TotalCallsiteSamples", i64 30}
+
+; CHECK-OBJ: .llvm_stats
+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
index 0e88dea3cdd02..c2c6dfa9c0f28 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
@@ -1,9 +1,16 @@
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -report-profile-staleness -S 2>%t
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll
; RUN: FileCheck %s --input-file %t
+; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD
+; RUN: llc < %t.ll -filetype=obj -o %t.obj
+; RUN: llvm-objdump --section-headers %t.obj | FileCheck %s --check-prefix=CHECK-OBJ
; CHECK: (1/3) of functions' profile are invalid and (10/50) of samples are discarded due to function hash mismatch.
; CHECK: (2/3) of callsites' profile are invalid and (20/30) of samples are discarded due to callsite location mismatch.
+; CHECK-MD: ![[#]] = !{!"NumMismatchedFuncHash", i64 1, !"TotalProfiledFunc", i64 3, !"MismatchedFuncHashSamples", i64 10, !"TotalFuncHashSamples", i64 50, !"MismatchedCallsiteSamples", i64 20, !"TotalCallsiteSamples", i64 30}
+
+; CHECK-OBJ: .llvm_stats
+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
More information about the llvm-commits
mailing list