[llvm] [CSProfileGenerator] Infer callsite samples for probeless inlined functions (PR #191572)
Wei Wang via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 10 21:00:38 PDT 2026
https://github.com/apolloww updated https://github.com/llvm/llvm-project/pull/191572
>From cfb8119f5f553dfd57aaa9dfcb0b52df98f69394 Mon Sep 17 00:00:00 2001
From: Wei Wang <apollo.mobility at gmail.com>
Date: Fri, 10 Apr 2026 13:25:42 -0700
Subject: [PATCH 1/2] [CSProfileGenerator] Infer callsite samples for probeless
inlined functions
In probe-based CSSPGO, some inlined functions (thin wrappers like
destructors, copy constructors, forwarding functions) are fully
optimized away after inlining, leaving no probes in the binary. Without
probes, they get zero body samples and no FunctionSamples, making them
invisible to the pre-inliner.
This patch adds three coordinated fixes:
1. inferCallsiteSamples() in ProfileGenerator.cpp: walks the context
trie bottom-up and creates FunctionSamples for probeless nodes by
inferring callsite body samples and called targets from callee entry
(head) samples. Guarded by --infer-callsite-samples (default: off).
2. trackInlineesOptimizedAway() in ProfiledBinary.cpp: extend to handle
inline tree nodes that have no probes but still have children in the
inline tree. This marks probeless inlined wrappers as size=0 so the
pre-inliner's size budget accounts for them correctly.
3. shouldInline() in CSPreInliner.cpp: change threshold check from
`SizeCost < SampleThreshold` to `SizeCost <= SampleThreshold`. The
probeless functions from (2) have SizeCost=0, and cold callsites
can have SampleThreshold=0, so the strict `<` incorrectly rejects
them (0 < 0 is false). The `<=` fix allows these zero-cost inlinees
through (0 <= 0 is true).
---
.../llvm-profgen/infer-callsite-samples.test | 21 ++++
llvm/tools/llvm-profgen/CSPreInliner.cpp | 2 +-
llvm/tools/llvm-profgen/ProfileGenerator.cpp | 116 +++++++++++++++++-
llvm/tools/llvm-profgen/ProfileGenerator.h | 5 +
llvm/tools/llvm-profgen/ProfiledBinary.cpp | 9 +-
5 files changed, 143 insertions(+), 10 deletions(-)
create mode 100644 llvm/test/tools/llvm-profgen/infer-callsite-samples.test
diff --git a/llvm/test/tools/llvm-profgen/infer-callsite-samples.test b/llvm/test/tools/llvm-profgen/infer-callsite-samples.test
new file mode 100644
index 0000000000000..afcdcd0a03485
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/infer-callsite-samples.test
@@ -0,0 +1,21 @@
+; Test inferCallsiteSamples with --infer-callsite-samples
+
+; Baseline: no inference
+; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-pseudoprobe.perfscript --binary=%S/Inputs/inline-cs-pseudoprobe.perfbin --output=%t1 --profile-summary-cold-count=0 --csspgo-preinliner=0 --gen-cs-nested-profile=0 --infer-callsite-samples=0
+; RUN: FileCheck %s --input-file %t1 --check-prefix=CHECK-BASELINE
+
+; With probeless inference: creates FunctionSamples for probeless main
+; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-pseudoprobe.perfscript --binary=%S/Inputs/inline-cs-pseudoprobe.perfbin --output=%t2 --profile-summary-cold-count=0 --csspgo-preinliner=0 --gen-cs-nested-profile=0 --infer-callsite-samples=1
+; RUN: FileCheck %s --input-file %t2 --check-prefix=CHECK-INFER
+
+; Baseline: main does not appear, foo total is 74
+; CHECK-BASELINE: [main:2 @ foo]:74:0
+; CHECK-BASELINE: [main:2 @ foo:8 @ bar]:28:14
+; CHECK-BASELINE-NOT: [main]:
+
+; Infer: main gets FunctionSamples inferred from foo's head samples.
+; CHECK-INFER: [main:2 @ foo]:74:0
+; CHECK-INFER: [main:2 @ foo:8 @ bar]:28:14
+; CHECK-INFER: [main]:1:0
+; CHECK-INFER-NEXT: 2: 1 foo:1
+; CHECK-INFER-NEXT: !CFGChecksum: 281479271677951
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp
index 87df6996aa435..e614f12785e74 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.cpp
+++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp
@@ -192,7 +192,7 @@ bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) {
SampleThreshold *= CSPreinlMultiplierForPrevInl;
}
- return (Candidate.SizeCost < SampleThreshold);
+ return (Candidate.SizeCost <= SampleThreshold);
}
void CSPreInliner::processFunction(const FunctionId Name) {
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index c3f489040007e..7c85203590099 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -119,6 +119,14 @@ cl::opt<bool> InferMissingFrames(
"Infer missing call frames due to compiler tail call elimination."),
cl::Optional, cl::cat(ProfGenCategory));
+cl::opt<bool> InferCallsiteSamples(
+ "infer-callsite-samples", llvm::cl::init(false),
+ llvm::cl::desc(
+ "Create FunctionSamples for probeless context trie nodes by inferring "
+ "callsite body samples from callee entry samples. Also match "
+ "under-counted callsite samples with callee entry samples."),
+ llvm::cl::Optional);
+
namespace sampleprof {
// Initialize the MaxCompressionSize to -1 which means no size limit
@@ -936,6 +944,8 @@ void CSProfileGenerator::generateProfile() {
if (SampleCounters) {
if (Binary->usePseudoProbes()) {
generateProbeBasedProfile();
+ if (InferCallsiteSamples)
+ inferCallsiteSamples();
} else {
generateLineNumBasedProfile();
}
@@ -1058,16 +1068,14 @@ void CSProfileGenerator::populateBoundarySamplesForFunction(
CallerNode->getFunctionSamples()->addCalledTargetSamples(
LeafLoc->Location.LineOffset,
getBaseDiscriminator(LeafLoc->Location.Discriminator),
- FunctionId(CalleeName),
- Count);
+ FunctionId(CalleeName), Count);
// Record head sample for called target(callee)
CalleeCallSite = LeafLoc->Location;
}
}
- ContextTrieNode *CalleeNode =
- CallerNode->getOrCreateChildContext(CalleeCallSite,
- FunctionId(CalleeName));
+ ContextTrieNode *CalleeNode = CallerNode->getOrCreateChildContext(
+ CalleeCallSite, FunctionId(CalleeName));
FunctionSamples *CalleeProfile = getOrCreateFunctionSamples(CalleeNode);
CalleeProfile->addHeadSamples(Count);
}
@@ -1321,7 +1329,8 @@ void CSProfileGenerator::populateBodySamplesWithProbes(
// context id to infer caller's context id to ensure they share the
// same context prefix.
uint64_t CallerIndex = ContextNode->getCallSiteLoc().LineOffset;
- uint64_t CallerDiscriminator = ContextNode->getCallSiteLoc().Discriminator;
+ uint64_t CallerDiscriminator =
+ ContextNode->getCallSiteLoc().Discriminator;
assert(CallerIndex &&
"Inferred caller's location index shouldn't be zero!");
assert(!CallerDiscriminator &&
@@ -1373,6 +1382,101 @@ void CSProfileGenerator::populateBoundarySamplesWithProbes(
}
}
+void CSProfileGenerator::inferCallsiteSamples() {
+ // Walk the context trie bottom-up (children before parents) and for each
+ // node, ensure that callsite body samples are at least as large as callee
+ // entry (head) samples. This serves two purposes:
+ // 1. Creates FunctionSamples for probeless nodes (those with no LBR samples
+ // on their probes, typically thin wrapper functions) using children's
+ // head samples.
+ // 2. Fixes under-counted callsites for nodes that have FunctionSamples but
+ // where LBR sparsity caused callsite counts to be lower than callee entry
+ // counts.
+ SmallVector<ContextTrieNode *, 256> AllNodes;
+ for (auto *Node : ContextTracker)
+ AllNodes.push_back(Node);
+
+ for (auto It = AllNodes.rbegin(), End = AllNodes.rend(); It != End; ++It) {
+ ContextTrieNode *Node = *It;
+
+ // Skip the synthetic root node.
+ if (Node == &getRootContext())
+ continue;
+
+ bool HasChildWithSamples = false;
+ for (auto &Child : Node->getAllChildContext()) {
+ if (Child.second.getFunctionSamples()) {
+ HasChildWithSamples = true;
+ break;
+ }
+ }
+ if (!HasChildWithSamples)
+ continue;
+
+ // Create FunctionSamples for probeless nodes.
+ FunctionSamples *FSamples = Node->getFunctionSamples();
+ if (!FSamples) {
+ FSamplesList.emplace_back();
+ FSamples = &FSamplesList.back();
+ FSamples->setFunction(Node->getFuncName());
+ // Set the CFGChecksum from the probe descriptor. The GUID lookup is
+ // safe because trie node names originate from probe descriptors which
+ // use canonicalized source function names.
+ const auto *FuncDesc = Binary->getFuncDescForGUID(
+ Node->getFuncName().getHashCode());
+ FSamples->setFunctionHash(FuncDesc->FuncHash);
+ Node->setFunctionSamples(FSamples);
+ }
+
+ // Aggregate callee head samples per callsite, since multiple callees
+ // (e.g., indirect calls) can share the same callsite location. Also add
+ // each callee as a called target.
+ std::map<LineLocation, uint64_t> CallsiteHeadSum;
+ for (auto &Child : Node->getAllChildContext()) {
+ FunctionSamples *ChildSamples = Child.second.getFunctionSamples();
+ if (!ChildSamples)
+ continue;
+ uint64_t ChildHeadSamples = ChildSamples->getHeadSamplesEstimate();
+ if (ChildHeadSamples == 0)
+ continue;
+ LineLocation ChildCallsite = Child.second.getCallSiteLoc();
+ // Only add the delta to avoid double-counting if there are already
+ // called target samples from normal profile generation.
+ uint64_t ExistingTargetSamples = 0;
+ auto ExistingTargets = FSamples->findCallTargetMapAt(
+ ChildCallsite.LineOffset, ChildCallsite.Discriminator);
+ if (ExistingTargets) {
+ auto TargetIt = ExistingTargets->find(Child.second.getFuncName());
+ if (TargetIt != ExistingTargets->end())
+ ExistingTargetSamples = TargetIt->second;
+ }
+ // Use the max of head samples and existing target samples to compute
+ // the expected contribution of this callee to the callsite total.
+ CallsiteHeadSum[ChildCallsite] +=
+ std::max(ChildHeadSamples, ExistingTargetSamples);
+ if (ChildHeadSamples > ExistingTargetSamples)
+ FSamples->addCalledTargetSamples(ChildCallsite.LineOffset,
+ ChildCallsite.Discriminator,
+ Child.second.getFuncName(),
+ ChildHeadSamples - ExistingTargetSamples);
+ }
+
+ // Ensure callsite body samples >= sum of callee head samples.
+ for (auto &[Loc, HeadSum] : CallsiteHeadSum) {
+ uint64_t ExistingSamples = 0;
+ auto Existing = FSamples->findSamplesAt(Loc.LineOffset,
+ Loc.Discriminator);
+ if (Existing)
+ ExistingSamples = *Existing;
+ if (HeadSum > ExistingSamples) {
+ uint64_t Delta = HeadSum - ExistingSamples;
+ FSamples->addBodySamples(Loc.LineOffset, Loc.Discriminator, Delta);
+ FSamples->addTotalSamples(Delta);
+ }
+ }
+ }
+}
+
ContextTrieNode *CSProfileGenerator::getContextNodeForLeafProbe(
const AddrBasedCtxKey *CtxKey, const MCDecodedPseudoProbe *LeafProbe) {
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
index 2c512779319d4..844dd33fccc5f 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.h
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -359,6 +359,11 @@ class CSProfileGenerator : public ProfileGeneratorBase {
getFunctionProfileForLeafProbe(const AddrBasedCtxKey *CtxKey,
const MCDecodedPseudoProbe *LeafProbe);
+ // For context trie nodes, infer callsite body samples from callee entry
+ // (head) samples. Creates FunctionSamples for probeless nodes and fixes
+ // under-counted callsites due to LBR sampling sparsity.
+ void inferCallsiteSamples();
+
void convertToProfileMap(ContextTrieNode &Node,
SampleContextFrameVector &Context);
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
index 915e991e4068c..4a9133f1a793f 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
@@ -158,9 +158,12 @@ void BinarySizeContextTracker::trackInlineesOptimizedAway(
ProbeDecoder.getFuncDescForGUID(ProbeNode.Guid)->FuncName;
ProbeContext.emplace_back(FuncName, 0);
- // This ProbeContext has a probe, so it has code before inlining and
- // optimization. Make sure we mark its size as known.
- if (!ProbeNode.getProbes().empty()) {
+ // Mark the size of this inlinee as known (zero) if it was optimized away.
+ // A function was inlined and then optimized if it either:
+ // 1. Still has probes but no instructions (original case), or
+ // 2. Has no probes left but still has children in the inline tree,
+ // meaning it existed as a real function before optimization.
+ if (!ProbeNode.getProbes().empty() || !ProbeNode.getChildren().empty()) {
ContextTrieNode *SizeContext = &RootContext;
for (auto &ProbeFrame : reverse(ProbeContext)) {
StringRef CallerName = ProbeFrame.first;
>From 5e24e5026ed6125c7fb6075e6a240cf052cfaab5 Mon Sep 17 00:00:00 2001
From: Wei Wang <apollo.mobility at gmail.com>
Date: Fri, 10 Apr 2026 20:57:04 -0700
Subject: [PATCH 2/2] use getOrCreateFunctionSamples
---
llvm/tools/llvm-profgen/ProfileGenerator.cpp | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index 7c85203590099..f5334ca1a7f88 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -1416,16 +1416,13 @@ void CSProfileGenerator::inferCallsiteSamples() {
// Create FunctionSamples for probeless nodes.
FunctionSamples *FSamples = Node->getFunctionSamples();
if (!FSamples) {
- FSamplesList.emplace_back();
- FSamples = &FSamplesList.back();
- FSamples->setFunction(Node->getFuncName());
+ FSamples = getOrCreateFunctionSamples(Node);
// Set the CFGChecksum from the probe descriptor. The GUID lookup is
// safe because trie node names originate from probe descriptors which
// use canonicalized source function names.
const auto *FuncDesc = Binary->getFuncDescForGUID(
Node->getFuncName().getHashCode());
FSamples->setFunctionHash(FuncDesc->FuncHash);
- Node->setFunctionSamples(FSamples);
}
// Aggregate callee head samples per callsite, since multiple callees
More information about the llvm-commits
mailing list