[llvm-branch-commits] [llvm] [llvm-profgen] Account for indirect-call promotion limits in the CS pre-inliner (PR #243631)
Kunal Pathak via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Nov 17 14:28:36 PST 2026
https://github.com/kunalspathak updated https://github.com/llvm/llvm-project/pull/243631
>From 5053ad3afc26568caa99af5bb2391c5d92260e0a Mon Sep 17 00:00:00 2001
From: Kunal Pathak <kupathak at meta.com>
Date: Tue, 17 Nov 2026 13:26:47 -0800
Subject: [PATCH 1/2] [llvm-profgen] Respect the sample loader's indirect-call
promotion limits
The CS pre-inliner can plan to inline a target that the sample loader
will not promote. Keep rejected contexts in the callee's base profile
instead of spending the caller's inline budget on them.
Read the loader's hotness, initial exemption, and maximum-promotion
options. Account for call-target counts being replaced by nested head
estimates as targets are accepted. Leave the compiler thresholds intact.
Apply the gate only to sites with an indirect-call pseudo probe. Collect
that evidence during the existing decoder pass, including functions and
inlinees skipped by the sample GUID filter. Add a decoder callback test
and profile tests for the promotion limits and call-probe checks.
This follows the compiler-side recovery change, which also handles hot
contexts already nested in existing generated profiles.
Assisted-by: GPT-6 Astra
---
llvm/include/llvm/MC/MCPseudoProbe.h | 14 +-
.../llvm/Transforms/IPO/SampleProfile.h | 7 +
llvm/lib/MC/MCPseudoProbe.cpp | 16 +-
llvm/lib/Transforms/IPO/SampleProfile.cpp | 16 ++
.../X86/Inputs/cs-preinline-icp-map-head.prof | 48 ++++++
.../Inputs/cs-preinline-icp-probe-type.prof | 53 +++++++
.../X86/Inputs/cs-preinline-icp.prof | 48 ++++++
.../llvm-profgen/X86/cs-preinline-icp.test | 100 +++++++++++++
llvm/tools/llvm-profgen/CSPreInliner.cpp | 139 +++++++++++++++++-
llvm/tools/llvm-profgen/CSPreInliner.h | 10 ++
llvm/tools/llvm-profgen/ProfiledBinary.cpp | 12 +-
llvm/tools/llvm-profgen/ProfiledBinary.h | 15 ++
llvm/unittests/MC/MCPseudoProbeTest.cpp | 33 +++++
13 files changed, 500 insertions(+), 11 deletions(-)
create mode 100644 llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp-map-head.prof
create mode 100644 llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp-probe-type.prof
create mode 100644 llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp.prof
create mode 100644 llvm/test/tools/llvm-profgen/X86/cs-preinline-icp.test
diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h
index fe63137a3bf166..41e810043615dc 100644
--- a/llvm/include/llvm/MC/MCPseudoProbe.h
+++ b/llvm/include/llvm/MC/MCPseudoProbe.h
@@ -57,6 +57,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator.h"
@@ -436,6 +437,10 @@ class MCPseudoProbeDecoder {
using Uint64Set = DenseSet<uint64_t>;
using Uint64Map = DenseMap<uint64_t, uint64_t>;
+ // Called with the GUID of the function that owns a probe, the probe index and
+ // the probe type.
+ using ProbeCallback =
+ function_ref<void(uint64_t Guid, uint32_t Index, PseudoProbeType Type)>;
// Decode pseudo_probe_desc section to build GUID to PseudoProbeFuncDesc map.
// If pseudo_probe_desc section is mapped to memory and \p IsMMapped is true,
@@ -451,10 +456,12 @@ class MCPseudoProbeDecoder {
const Uint64Set &GuidFilter);
// Decode pseudo_probe section to build address to probes map for specifed
- // functions only.
+ // functions only. If \p Callback is set, also call it for every probe in the
+ // section, including the probes of functions that the GUID filter skips.
LLVM_ABI bool buildAddress2ProbeMap(const uint8_t *Start, std::size_t Size,
const Uint64Set &GuildFilter,
- const Uint64Map &FuncStartAddrs);
+ const Uint64Map &FuncStartAddrs,
+ ProbeCallback Callback = nullptr);
// Print pseudo_probe_desc section info
LLVM_ABI void printGUID2FuncDescMap(raw_ostream &OS);
@@ -530,7 +537,8 @@ class MCPseudoProbeDecoder {
bool buildAddress2ProbeMap(MCDecodedPseudoProbeInlineTree *Cur,
uint64_t &LastAddr, const Uint64Set &GuildFilter,
const Uint64Map &FuncStartAddrs,
- const uint32_t CurChildIndex);
+ const uint32_t CurChildIndex,
+ ProbeCallback Callback);
};
} // end namespace llvm
diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfile.h b/llvm/include/llvm/Transforms/IPO/SampleProfile.h
index 085b50f18c4686..3c5a9fefd1a3cb 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleProfile.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleProfile.h
@@ -33,6 +33,13 @@ LLVM_ABI extern cl::opt<int> ProfileInlineGrowthLimit;
LLVM_ABI extern cl::opt<int> ProfileInlineLimitMin;
LLVM_ABI extern cl::opt<int> ProfileInlineLimitMax;
LLVM_ABI extern cl::opt<bool> SortProfiledSCC;
+/// The promotion limits used by the sample loader and llvm-profgen.
+LLVM_ABI unsigned
+getSampleProfileICPRelativeHotness(const cl::LibraryOptionsContext &Opts);
+LLVM_ABI unsigned
+getSampleProfileICPRelativeHotnessSkip(const cl::LibraryOptionsContext &Opts);
+LLVM_ABI unsigned
+getSampleProfileICPMaxProm(const cl::LibraryOptionsContext &Opts);
/// The sample profiler data loader pass.
class SampleProfileLoaderPass
diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp
index 85bfde80722eb7..bb020a6b9e305a 100644
--- a/llvm/lib/MC/MCPseudoProbe.cpp
+++ b/llvm/lib/MC/MCPseudoProbe.cpp
@@ -464,7 +464,7 @@ template <bool IsTopLevelFunc>
bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
MCDecodedPseudoProbeInlineTree *Cur, uint64_t &LastAddr,
const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs,
- const uint32_t CurChildIndex) {
+ const uint32_t CurChildIndex, ProbeCallback Callback) {
// The pseudo_probe section encodes an inline forest and each tree has a
// format defined in MCPseudoProbe.h
@@ -544,6 +544,10 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
Discriminator, Cur);
++CurrentProbeCount;
}
+ // The probes of skipped functions are read anyway to find the next record,
+ // so the callback can see them at no extra decoding cost.
+ if (Callback && !isSentinelProbe(Attr))
+ Callback(Guid, Index, PseudoProbeType(Kind));
LastAddr = Addr;
}
@@ -555,7 +559,8 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
MutableArrayRef(InlineTreeVec).take_back(ChildrenToProcess);
}
for (uint32_t I = 0; I < ChildrenToProcess; I++) {
- buildAddress2ProbeMap<false>(Cur, LastAddr, GuidFilter, FuncStartAddrs, I);
+ buildAddress2ProbeMap<false>(Cur, LastAddr, GuidFilter, FuncStartAddrs, I,
+ Callback);
}
return Cur;
}
@@ -641,7 +646,7 @@ bool MCPseudoProbeDecoder::countRecords(bool &Discard, uint32_t &ProbeCount,
bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter,
- const Uint64Map &FuncStartAddrs) {
+ const Uint64Map &FuncStartAddrs, ProbeCallback Callback) {
// For function records in the order of their appearance in the encoded data
// (DFS), count the number of contained probes and inlined function records.
uint32_t ProbeCount = 0;
@@ -668,8 +673,9 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
uint64_t LastAddr = 0;
uint32_t CurChildIndex = 0;
while (Data < End)
- CurChildIndex += buildAddress2ProbeMap<true>(
- &DummyInlineRoot, LastAddr, GuidFilter, FuncStartAddrs, CurChildIndex);
+ CurChildIndex +=
+ buildAddress2ProbeMap<true>(&DummyInlineRoot, LastAddr, GuidFilter,
+ FuncStartAddrs, CurChildIndex, Callback);
assert(Data == End && "Have unprocessed data in pseudo_probe section");
assert(PseudoProbeVec.size() == ProbeCount &&
"Mismatching probe count pre- and post-parsing");
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index efd858c19ced54..62ebead633455e 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -66,6 +66,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/LibraryOptionsContext.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/IPO.h"
@@ -146,6 +147,21 @@ cl::opt<int> SampleColdCallSiteThreshold(
cl::desc("Threshold for inlining cold callsites"));
} // namespace llvm
+unsigned llvm::getSampleProfileICPRelativeHotness(
+ const cl::LibraryOptionsContext &Opts) {
+ return Opts.get<IPOOptions>().sample_profile_icp_relative_hotness;
+}
+
+unsigned llvm::getSampleProfileICPRelativeHotnessSkip(
+ const cl::LibraryOptionsContext &Opts) {
+ return Opts.get<IPOOptions>().sample_profile_icp_relative_hotness_skip;
+}
+
+unsigned
+llvm::getSampleProfileICPMaxProm(const cl::LibraryOptionsContext &Opts) {
+ return Opts.get<IPOOptions>().sample_profile_icp_max_prom;
+}
+
namespace llvm {
extern cl::opt<bool> EnableExtTspBlockPlacement;
}
diff --git a/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp-map-head.prof b/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp-map-head.prof
new file mode 100644
index 00000000000000..33e8dbdb6fa2fc
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp-map-head.prof
@@ -0,0 +1,48 @@
+[quick_sort]:2000:1000
+ 1: 1000
+ 4: 1000 t1:600 t2:200 t3:200
+ !CFGChecksum: 1
+[quick_sort:4 @ t1]:6000:600
+ 1: 600
+ 2: 5400
+ !CFGChecksum: 2
+[quick_sort:4 @ t2]:2000:260
+ 1: 260
+ 2: 1740
+ !CFGChecksum: 3
+[quick_sort:4 @ t3]:2000:200
+ 1: 200
+ 2: 1800
+ !CFGChecksum: 4
+[cold]:60:1
+ 1: 2
+ 2: 2
+ 3: 2
+ 4: 2
+ 5: 2
+ 6: 2
+ 7: 2
+ 8: 2
+ 9: 2
+ 10: 2
+ 11: 2
+ 12: 2
+ 13: 2
+ 14: 2
+ 15: 2
+ 16: 2
+ 17: 2
+ 18: 2
+ 19: 2
+ 20: 2
+ 21: 2
+ 22: 2
+ 23: 2
+ 24: 2
+ 25: 2
+ 26: 2
+ 27: 2
+ 28: 2
+ 29: 2
+ 30: 2
+ !CFGChecksum: 5
diff --git a/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp-probe-type.prof b/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp-probe-type.prof
new file mode 100644
index 00000000000000..5fa771f3cdc18e
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp-probe-type.prof
@@ -0,0 +1,53 @@
+[quick_sort]:3100:2000
+ 1: 2000
+ 4: 100 t1:100
+ 5: 1000 t2:600 t3:200 t4:200
+ !CFGChecksum: 1
+[quick_sort:4 @ t1]:19100:100
+ 1: 100
+ 2: 19000
+ !CFGChecksum: 2
+[quick_sort:5 @ t2]:6000:600
+ 1: 600
+ 2: 5400
+ !CFGChecksum: 3
+[quick_sort:5 @ t3]:2000:200
+ 1: 200
+ 2: 1800
+ !CFGChecksum: 4
+[quick_sort:5 @ t4]:2000:200
+ 1: 200
+ 2: 1800
+ !CFGChecksum: 5
+[cold]:60:1
+ 1: 2
+ 2: 2
+ 3: 2
+ 4: 2
+ 5: 2
+ 6: 2
+ 7: 2
+ 8: 2
+ 9: 2
+ 10: 2
+ 11: 2
+ 12: 2
+ 13: 2
+ 14: 2
+ 15: 2
+ 16: 2
+ 17: 2
+ 18: 2
+ 19: 2
+ 20: 2
+ 21: 2
+ 22: 2
+ 23: 2
+ 24: 2
+ 25: 2
+ 26: 2
+ 27: 2
+ 28: 2
+ 29: 2
+ 30: 2
+ !CFGChecksum: 6
diff --git a/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp.prof b/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp.prof
new file mode 100644
index 00000000000000..ad69bb20d77b2c
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/X86/Inputs/cs-preinline-icp.prof
@@ -0,0 +1,48 @@
+[quick_sort]:2000:1000
+ 1: 1000
+ 4: 1000 t1:600 t2:200 t3:200
+ !CFGChecksum: 1
+[quick_sort:4 @ t1]:6000:600
+ 1: 600
+ 2: 5400
+ !CFGChecksum: 2
+[quick_sort:4 @ t2]:2000:200
+ 1: 200
+ 2: 1800
+ !CFGChecksum: 3
+[quick_sort:4 @ t3]:2000:200
+ 1: 200
+ 2: 1800
+ !CFGChecksum: 4
+[cold]:60:1
+ 1: 2
+ 2: 2
+ 3: 2
+ 4: 2
+ 5: 2
+ 6: 2
+ 7: 2
+ 8: 2
+ 9: 2
+ 10: 2
+ 11: 2
+ 12: 2
+ 13: 2
+ 14: 2
+ 15: 2
+ 16: 2
+ 17: 2
+ 18: 2
+ 19: 2
+ 20: 2
+ 21: 2
+ 22: 2
+ 23: 2
+ 24: 2
+ 25: 2
+ 26: 2
+ 27: 2
+ 28: 2
+ 29: 2
+ 30: 2
+ !CFGChecksum: 5
diff --git a/llvm/test/tools/llvm-profgen/X86/cs-preinline-icp.test b/llvm/test/tools/llvm-profgen/X86/cs-preinline-icp.test
new file mode 100644
index 00000000000000..e7b359538d5620
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/X86/cs-preinline-icp.test
@@ -0,0 +1,100 @@
+; The preinliner must not plan inlining of an indirect call target that the
+; sample profile loader will not promote (top target, then >= 25% of the site,
+; at most 3 targets by default). Such contexts are merged into the callee's base
+; profile instead of being stranded at the call site. In
+; fs-discriminator-probe.perfbin, probe 4 of quick_sort is an indirect call.
+
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --output=%t1
+; RUN: FileCheck %s --input-file %t1 --check-prefix=CHECK-ICP
+
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --csspgo-preinliner-honor-icp-limits=0 --output=%t2
+; RUN: FileCheck %s --input-file %t2 --check-prefix=CHECK-ALL
+
+; A lower ICP threshold lets the 20% targets be promoted, so they stay inlined.
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-icp-relative-hotness=20 --output=%t3
+; RUN: FileCheck %s --input-file %t3 --check-prefix=CHECK-ALL
+
+; Exempting all three targets from the percentage check also keeps them inlined.
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-icp-relative-hotness-skip=3 --output=%t-skip
+; RUN: FileCheck %s --input-file %t-skip --check-prefix=CHECK-ALL
+
+; The loader also promotes at most sample-profile-icp-max-prom targets per call.
+; With the percentage check at 20%, all three targets pass it, so a limit of 2
+; rejects t3, and a limit of 0 rejects all three.
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-icp-relative-hotness=20 --sample-profile-icp-max-prom=2 --output=%t-maxprom
+; RUN: FileCheck %s --input-file %t-maxprom --check-prefix=CHECK-MAXPROM --implicit-check-not="@ t3]"
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-icp-relative-hotness=20 --sample-profile-icp-max-prom=0 --output=%t-noprom
+; RUN: FileCheck %s --input-file %t-noprom --check-prefix=CHECK-NOPROM --implicit-check-not="[quick_sort:4 @"
+
+; CHECK-MAXPROM-DAG: [quick_sort:4 @ t1]:6000:600
+; CHECK-MAXPROM-DAG: [quick_sort:4 @ t2]:2000:200
+; CHECK-MAXPROM-DAG: [t3]:2000:200
+
+; CHECK-NOPROM-DAG: [t1]:6000:600
+; CHECK-NOPROM-DAG: [t2]:2000:200
+; CHECK-NOPROM-DAG: [t3]:2000:200
+
+; A site without an indirect call probe is a direct call, even with several
+; targets: a direct call site can show several functions when the binary lost
+; the inline frames between them. inline-cs-pseudoprobe.perfbin has no function
+; quick_sort, so quick_sort:4 has no call probe there and all three targets are
+; pre-inlined.
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/inline-cs-pseudoprobe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --output=%t-noprobe
+; RUN: FileCheck %s --input-file %t-noprobe --check-prefix=CHECK-ALL
+
+; The nested profile consumed by the compiler keeps the full samples of the
+; rejected targets in their own profiles.
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=1 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --output=%t4
+; RUN: FileCheck %s --input-file %t4 --check-prefix=CHECK-NESTED
+
+; CHECK-ICP: [quick_sort:4 @ t1]:6000:600
+; CHECK-ICP: !Attributes: 2
+; CHECK-ICP: [quick_sort]:2000:1000
+; CHECK-ICP-NEXT: 1: 1000
+; CHECK-ICP-NEXT: 4: 1000 t1:600 t2:200 t3:200
+; CHECK-ICP-NOT: [quick_sort:4 @ t2]
+; CHECK-ICP-NOT: [quick_sort:4 @ t3]
+; CHECK-ICP: [t2]:2000:200
+; CHECK-ICP: [t3]:2000:200
+
+; CHECK-ALL-DAG: [quick_sort:4 @ t1]:6000:600
+; CHECK-ALL-DAG: [quick_sort:4 @ t2]:2000:200
+; CHECK-ALL-DAG: [quick_sort:4 @ t3]:2000:200
+; CHECK-ALL-NOT: [t2]:
+; CHECK-ALL-NOT: [t3]:
+
+; CHECK-NESTED: quick_sort:7400:1000
+; CHECK-NESTED-NEXT: 1: 1000
+; CHECK-NESTED-NEXT: 4: 400 t2:200 t3:200
+; CHECK-NESTED-NEXT: 4: t1:6000
+; CHECK-NESTED: t2:2000:200
+; CHECK-NESTED: t3:2000:200
+
+; The loader's site total uses the nested head estimate of each inlined target
+; and the map count of each outlined one. Here t2's map count is 200 but its
+; head estimate is 260: inlining it would make the total 1060, and 260 is
+; below 25% of that, so t2 must not be pre-inlined either.
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp-map-head.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=1 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --output=%t5
+; RUN: FileCheck %s --input-file %t5 --check-prefix=CHECK-MAPHEAD
+
+; CHECK-MAPHEAD: quick_sort:7400:1000
+; CHECK-MAPHEAD-NEXT: 1: 1000
+; CHECK-MAPHEAD-NEXT: 4: 400 t2:200 t3:200
+; CHECK-MAPHEAD-NEXT: 4: t1:6000
+; CHECK-MAPHEAD-NOT: 4: t2:
+; CHECK-MAPHEAD: t2:2000:260
+; CHECK-MAPHEAD: t3:2000:200
+
+; Probe 5 of quick_sort is a direct call. The single target at the indirect
+; call is not hot enough for ICP, so it is not pre-inlined. The direct call
+; site is not subject to ICP, so all its targets are pre-inlined.
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp-probe-type.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --output=%t7
+; RUN: FileCheck %s --input-file %t7 --check-prefix=CHECK-PROBETYPE
+
+; CHECK-PROBETYPE-DAG: [t1]:19100:100
+; CHECK-PROBETYPE-DAG: [quick_sort:5 @ t2]:6000:600
+; CHECK-PROBETYPE-DAG: [quick_sort:5 @ t3]:2000:200
+; CHECK-PROBETYPE-DAG: [quick_sort:5 @ t4]:2000:200
+; CHECK-PROBETYPE-NOT: [quick_sort:4 @ t1]
+; CHECK-PROBETYPE-NOT: [t3]:
+; CHECK-PROBETYPE-NOT: [t4]:
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp
index 421552f9c6dbb9..feb553b6f235f2 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.cpp
+++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp
@@ -12,6 +12,7 @@
#include "llvm/ADT/Statistic.h"
#include "llvm/Transforms/IPO/SampleProfile.h"
#include <cstdint>
+#include <map>
#define DEBUG_TYPE "cs-preinliner"
@@ -22,6 +23,9 @@ STATISTIC(PreInlNumCSInlined,
"Number of functions inlined with context sensitive profile");
STATISTIC(PreInlNumCSNotInlined,
"Number of functions not inlined with context sensitive profile");
+STATISTIC(PreInlNumCSICPRejected,
+ "Number of indirect call target contexts not inlined because the "
+ "sample profile loader would not promote them");
STATISTIC(PreInlNumCSInlinedHitMinLimit,
"Number of functions with FDO inline stopped due to min size limit");
STATISTIC(PreInlNumCSInlinedHitMaxLimit,
@@ -49,6 +53,12 @@ static cl::opt<bool> SamplePreInlineReplay(
cl::desc(
"Replay previous inlining and adjust context profile accordingly"));
+static cl::opt<bool> CSPreinlHonorICPLimits(
+ "csspgo-preinliner-honor-icp-limits", cl::Hidden, cl::init(true),
+ cl::desc("Do not inline indirect call target contexts that the sample "
+ "profile loader will not promote (see "
+ "-sample-profile-icp-relative-hotness)."));
+
static cl::opt<int> CSPreinlMultiplierForPrevInl(
"csspgo-preinliner-multiplier-for-previous-inlining", cl::Hidden,
cl::init(100),
@@ -70,6 +80,12 @@ CSPreInliner::CSPreInliner(SampleContextTracker &Tracker,
SampleColdCallSiteThreshold = 0;
if (!ProfileInlineLimitMax.getNumOccurrences())
ProfileInlineLimitMax = 50000;
+
+ ProfileICPRelativeHotness =
+ getSampleProfileICPRelativeHotness(Binary.getLibraryOptions());
+ ProfileICPRelativeHotnessSkip =
+ getSampleProfileICPRelativeHotnessSkip(Binary.getLibraryOptions());
+ ProfileICPMaxProm = getSampleProfileICPMaxProm(Binary.getLibraryOptions());
}
std::vector<FunctionId> CSPreInliner::buildTopDownOrder() {
@@ -102,6 +118,116 @@ std::vector<FunctionId> CSPreInliner::buildTopDownOrder() {
return Order;
}
+// Find the indirect call targets under CallerNode that the sample loader won't
+// promote, so we don't plan to inline them. The loader (see
+// inlineHotFunctionsWithPriority) walks a call's targets from hottest to
+// coldest and promotes one only if it's hot, is at least
+// ProfileICPRelativeHotness percent of the site total (once the first
+// ProfileICPRelativeHotnessSkip targets are promoted), and fewer than
+// ProfileICPMaxProm targets have been promoted so far. Pre-inlining a target
+// that fails these checks just burns the caller's size budget and leaves its
+// samples nested at the call site instead of in its own profile.
+//
+// One wrinkle: nesting a target under the caller makes convertCSProfiles drop
+// its call-target entry, so the loader counts its head samples instead and the
+// site total moves. Before accepting a target, we re-check it and every target
+// accepted so far against the new total; if that fails, it and all colder
+// targets are rejected.
+DenseSet<const ContextTrieNode *>
+CSPreInliner::getICPRejectedCallees(ContextTrieNode *CallerNode,
+ const FunctionSamples *CallerSamples) {
+ DenseSet<const ContextTrieNode *> Rejected;
+ // Only a pseudo probe identifies a single call instruction. With line-based
+ // profiles, distinct direct calls on one source line share a call site, so
+ // several targets there do not imply an indirect call.
+ if (!CSPreinlHonorICPLimits || SamplePreInlineReplay ||
+ !FunctionSamples::ProfileIsProbeBased)
+ return Rejected;
+
+ struct Target {
+ ContextTrieNode *Node;
+ uint64_t HeadCount;
+ uint64_t CallTargetCount = 0;
+ };
+ std::map<LineLocation, SmallVector<Target, 4>> CallSites;
+ for (auto &Child : CallerNode->getAllChildContext()) {
+ ContextTrieNode *CalleeNode = &Child.second;
+ if (FunctionSamples *CalleeSamples = CalleeNode->getFunctionSamples())
+ CallSites[CalleeNode->getCallSiteLoc()].push_back(
+ {CalleeNode, CalleeSamples->getHeadSamplesEstimate()});
+ }
+
+ uint64_t HotCountThreshold = ProfileSummaryBuilder::getHotCountThreshold(
+ Summary->getDetailedSummary());
+ for (auto &Entry : CallSites) {
+ const LineLocation &CallSite = Entry.first;
+ // Only indirect calls are subject to ICP, so check the call probe type. It
+ // comes from the source call, and ICP keeps the original indirect call as
+ // the fallback, so a promoted site keeps its indirect call probe. A site
+ // without one is a direct call, even if several functions are recorded
+ // there (e.g. when the binary lost the inline frames between them).
+ if (!Binary.isIndirectCallProbe(CallerNode->getFuncName().getHashCode(),
+ CallSite.LineOffset))
+ continue;
+
+ auto &Callees = Entry.second;
+ // Site total with every target outlined, i.e. the raw call target map.
+ uint64_t SiteTotal = 0;
+ if (auto CallTargets = CallerSamples->findCallTargetMapAt(CallSite)) {
+ for (const auto &[Name, Count] : *CallTargets)
+ SiteTotal += Count;
+ for (Target &T : Callees) {
+ auto It = CallTargets->find(T.Node->getFuncName());
+ if (It != CallTargets->end())
+ T.CallTargetCount = It->second;
+ }
+ }
+
+ llvm::sort(Callees, [](const Target &L, const Target &R) {
+ if (L.HeadCount != R.HeadCount)
+ return L.HeadCount > R.HeadCount;
+ return L.Node->getFunctionSamples()->getGUID() <
+ R.Node->getFunctionSamples()->getGUID();
+ });
+ auto CanPromoteAll = [&](size_t NumCandidates, uint64_t NewTotal) {
+ for (size_t I = 0; I < NumCandidates; ++I) {
+ uint64_t HeadCount = Callees[I].HeadCount;
+ if (HeadCount < HotCountThreshold)
+ return false;
+ // The first few targets are exempt only from the percentage check.
+ if (I < ProfileICPRelativeHotnessSkip)
+ continue;
+ if (HeadCount * 100 < NewTotal * ProfileICPRelativeHotness)
+ return false;
+ }
+ return true;
+ };
+ LLVM_DEBUG(dbgs() << " Indirect call site " << CallSite.LineOffset << "."
+ << CallSite.Discriminator << ": " << Callees.size()
+ << " profiled targets, total count " << SiteTotal
+ << "\n");
+ size_t MaxAccepted = std::min<size_t>(Callees.size(), ProfileICPMaxProm);
+ size_t NumAccepted = 0;
+ for (; NumAccepted < MaxAccepted; ++NumAccepted) {
+ const Target &T = Callees[NumAccepted];
+ uint64_t NewTotal = SiteTotal - T.CallTargetCount + T.HeadCount;
+ // The new total must keep this target and the accepted prefix eligible.
+ if (!CanPromoteAll(NumAccepted + 1, NewTotal))
+ break;
+ SiteTotal = NewTotal;
+ }
+ for (size_t I = NumAccepted; I < Callees.size(); ++I) {
+ const Target &T = Callees[I];
+ Rejected.insert(T.Node);
+ LLVM_DEBUG(dbgs() << " Not promotable by ICP: "
+ << ContextTracker.getContextString(T.Node) << " (head "
+ << T.HeadCount << ", site total " << SiteTotal
+ << ")\n");
+ }
+ }
+ return Rejected;
+}
+
bool CSPreInliner::getInlineCandidates(ProfiledCandidateQueue &CQueue,
const FunctionSamples *CallerSamples) {
assert(CallerSamples && "Expect non-null caller samples");
@@ -112,6 +238,8 @@ bool CSPreInliner::getInlineCandidates(ProfiledCandidateQueue &CQueue,
// targets from function profile.
ContextTrieNode *CallerNode =
ContextTracker.getContextNodeForProfile(CallerSamples);
+ DenseSet<const ContextTrieNode *> ICPRejected =
+ getICPRejectedCallees(CallerNode, CallerSamples);
bool HasNewCandidate = false;
for (auto &Child : CallerNode->getAllChildContext()) {
@@ -135,8 +263,10 @@ bool CSPreInliner::getInlineCandidates(ProfiledCandidateQueue &CQueue,
// check for that.
HasNewCandidate = true;
uint32_t CalleeSize = getFuncSize(CalleeNode);
- CQueue.emplace(CalleeSamples, std::max(CallsiteCount, CalleeEntryCount),
- CalleeSize);
+ ProfiledInlineCandidate Candidate(
+ CalleeSamples, std::max(CallsiteCount, CalleeEntryCount), CalleeSize);
+ Candidate.ICPRejected = ICPRejected.contains(CalleeNode);
+ CQueue.push(Candidate);
}
return HasNewCandidate;
@@ -157,6 +287,11 @@ bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) {
if (SamplePreInlineReplay)
return WasInlined;
+ if (Candidate.ICPRejected) {
+ ++PreInlNumCSICPRejected;
+ return false;
+ }
+
unsigned int SampleThreshold = SampleColdCallSiteThreshold;
uint64_t ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold(
(Summary->getDetailedSummary()));
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.h b/llvm/tools/llvm-profgen/CSPreInliner.h
index 022c3f8d0daed0..4a587899c87498 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.h
+++ b/llvm/tools/llvm-profgen/CSPreInliner.h
@@ -10,6 +10,7 @@
#define LLVM_TOOLS_LLVM_PROFGEN_PGOINLINEADVISOR_H
#include "ProfiledBinary.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/ProfileData/SampleProf.h"
@@ -32,6 +33,9 @@ struct ProfiledInlineCandidate {
uint64_t CallsiteCount;
// Size proxy for function under particular call context.
uint64_t SizeCost;
+ // Indirect call target that SampleProfileLoader will not promote, so its
+ // context cannot be inlined and must be merged into the callee's base.
+ bool ICPRejected = false;
};
// Inline candidate comparer using call site weight
@@ -81,7 +85,13 @@ class CSPreInliner {
void processFunction(FunctionId Name);
bool shouldInline(ProfiledInlineCandidate &Candidate);
uint32_t getFuncSize(const ContextTrieNode *ContextNode);
+ DenseSet<const ContextTrieNode *>
+ getICPRejectedCallees(ContextTrieNode *CallerNode,
+ const FunctionSamples *CallerSamples);
bool UseContextCost;
+ unsigned ProfileICPRelativeHotness;
+ unsigned ProfileICPRelativeHotnessSkip;
+ unsigned ProfileICPMaxProm;
SampleContextTracker &ContextTracker;
ProfiledBinary &Binary;
ProfileSummary *Summary;
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
index 96566c7254c08f..d3a36fafea8caf 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
@@ -548,9 +548,19 @@ void ProfiledBinary::decodePseudoProbe(const ObjectFile *Obj) {
"Pseudo Probe decoder fail in .pseudo_probe_desc section");
} else if (SectionName == ".pseudo_probe") {
StringRef Contents = unwrapOrError(Section.getContents(), FileName);
+ // Record the indirect call probes of every function, not only of the
+ // decoded ones. A copy without samples is not decoded, but it can be the
+ // only copy that keeps the indirect call, e.g. when the other copies
+ // inlined its targets.
+ auto RecordIndirectCallProbe = [&](uint64_t Guid, uint32_t Index,
+ PseudoProbeType Type) {
+ if (Type == PseudoProbeType::IndirectCall)
+ IndirectCallProbes.insert({Guid, Index});
+ };
if (!ProbeDecoder.buildAddress2ProbeMap(
reinterpret_cast<const uint8_t *>(Contents.data()),
- Contents.size(), GuidFilter, FuncStartAddresses))
+ Contents.size(), GuidFilter, FuncStartAddresses,
+ RecordIndirectCallProbe))
exitWithError("Pseudo Probe decoder fail in .pseudo_probe section");
}
}
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h
index 516e0f7cdf5c1a..5a836f57a54367 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.h
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.h
@@ -342,6 +342,10 @@ class ProfiledBinary {
// Function name to probe frame map for top-level outlined functions.
StringMap<MCDecodedPseudoProbeInlineTree *> TopLevelProbeFrameMap;
+ // Indirect call probes by (function GUID, probe index), from every function
+ // in the binary. Filled by decodePseudoProbe.
+ DenseSet<std::pair<uint64_t, uint32_t>> IndirectCallProbes;
+
bool UseFSDiscriminator = false;
// Whether we need to symbolize all instructions to get function context size.
@@ -436,6 +440,10 @@ class ProfiledBinary {
const StringRef ExeBinPath, const StringRef DebugBinPath);
~ProfiledBinary();
+ const cl::LibraryOptionsContext &getLibraryOptions() const {
+ return LibraryOptions;
+ }
+
/// Decode the interesting parts of the binary and build internal data
/// structures. On high level, the parts of interest are:
/// 1. Text sections, including the main code section and the PLT
@@ -701,6 +709,13 @@ class ProfiledBinary {
return ProbeDecoder.getFuncDescForGUID(GUID);
}
+ // Returns whether a copy of function \p GUID in the binary has an indirect
+ // call probe with index \p Index. All functions are checked, including the
+ // ones whose probes were not decoded because they have no samples.
+ bool isIndirectCallProbe(uint64_t GUID, uint32_t Index) const {
+ return IndirectCallProbes.contains({GUID, Index});
+ }
+
const MCPseudoProbeFuncDesc *
getInlinerDescForProbe(const MCDecodedPseudoProbe *Probe) {
return ProbeDecoder.getInlinerDescForProbe(Probe);
diff --git a/llvm/unittests/MC/MCPseudoProbeTest.cpp b/llvm/unittests/MC/MCPseudoProbeTest.cpp
index 7e4472e55004d6..a0cd7d90e4bd87 100644
--- a/llvm/unittests/MC/MCPseudoProbeTest.cpp
+++ b/llvm/unittests/MC/MCPseudoProbeTest.cpp
@@ -50,4 +50,37 @@ TEST(GUIDProbeFunctionMapTest, FindInEmpty) {
EXPECT_EQ(Map.find(42), Map.end());
}
+// The callback of buildAddress2ProbeMap sees every probe in the section,
+// including the probes of a function that the GUID filter skips and of its
+// inlinees, while only the probes of the selected functions are decoded.
+TEST(MCPseudoProbeDecoderTest, CallbackSeesSkippedFunctions) {
+ // Function records as described in MCPseudoProbe.h: GUID, number of probes,
+ // number of inlinees, then for each probe its index, a byte with the probe
+ // type (0x80: an address delta follows) and the address delta. An inlinee
+ // record starts with the index of its inline site.
+ const uint8_t Section[] = {
+ // GUID 1: a block probe.
+ 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0x80, 4,
+ // GUID 2: a block probe and an inlinee at site 2.
+ 2, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0x80, 4,
+ // Inlinee GUID 3: an indirect call probe with index 2.
+ 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0x81, 4};
+ MCPseudoProbeDecoder Decoder;
+ MCPseudoProbeDecoder::Uint64Set GuidFilter;
+ GuidFilter.insert(1);
+ MCPseudoProbeDecoder::Uint64Map FuncStartAddrs;
+ std::vector<std::tuple<uint64_t, uint32_t, PseudoProbeType>> Seen;
+ ASSERT_TRUE(Decoder.buildAddress2ProbeMap(
+ Section, sizeof(Section), GuidFilter, FuncStartAddrs,
+ [&](uint64_t Guid, uint32_t Index, PseudoProbeType Type) {
+ Seen.emplace_back(Guid, Index, Type);
+ }));
+ EXPECT_EQ(Decoder.getAddress2ProbesMap().size(), 1u);
+ std::vector<std::tuple<uint64_t, uint32_t, PseudoProbeType>> Expected = {
+ {1, 1, PseudoProbeType::Block},
+ {2, 1, PseudoProbeType::Block},
+ {3, 2, PseudoProbeType::IndirectCall}};
+ EXPECT_EQ(Seen, Expected);
+}
+
} // namespace
>From d0ff0f64f9a10edaf95624b0085b0267ef0fb2a9 Mon Sep 17 00:00:00 2001
From: Kunal Pathak <kupathak at meta.com>
Date: Tue, 17 Nov 2026 14:18:37 -0800
Subject: [PATCH 2/2] [llvm-profgen] Cache and explain the pre-inliner hot
threshold
Compute the hot threshold once in the CSPreInliner constructor instead
of recomputing it for every caller. The input summary stays fixed for
the pre-inliner run.
Document why the ICP checks keep the contextless summary. The loader's
nested summary depends on the inline decisions and can give a lower
threshold. Using a lower estimate could let more indirect targets spend
the caller's inline size budget, leaving less room for other callees.
Rejected contexts instead follow the existing base-profile merge path.
The loader continues to apply its own hotness check.
Clarify nearby comments and diagnostics so a pre-inliner rejection is
not presented as an exact prediction of the loader. Add coverage for an
explicit hot-count override. Profile selection is unchanged.
Assisted-by: GPT-6 Astra
---
.../llvm-profgen/X86/cs-preinline-icp.test | 6 ++++
llvm/tools/llvm-profgen/CSPreInliner.cpp | 34 +++++++++++++------
llvm/tools/llvm-profgen/CSPreInliner.h | 5 +--
3 files changed, 32 insertions(+), 13 deletions(-)
diff --git a/llvm/test/tools/llvm-profgen/X86/cs-preinline-icp.test b/llvm/test/tools/llvm-profgen/X86/cs-preinline-icp.test
index e7b359538d5620..b99b2b043dae96 100644
--- a/llvm/test/tools/llvm-profgen/X86/cs-preinline-icp.test
+++ b/llvm/test/tools/llvm-profgen/X86/cs-preinline-icp.test
@@ -26,6 +26,12 @@
; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-icp-relative-hotness=20 --sample-profile-icp-max-prom=0 --output=%t-noprom
; RUN: FileCheck %s --input-file %t-noprom --check-prefix=CHECK-NOPROM --implicit-check-not="[quick_sort:4 @"
+; The cached hot threshold honors an explicit override. Even the hottest target
+; (600 samples) is rejected when the threshold is 601, preserving each target's
+; standalone profile.
+; RUN: llvm-profgen --format=text --llvm-sample-profile=%S/Inputs/cs-preinline-icp.prof --binary=%S/Inputs/fs-discriminator-probe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --use-context-cost-for-preinliner=0 --sample-profile-hot-inline-threshold=3000 --profile-summary-hot-count=601 --output=%t-hot-count
+; RUN: FileCheck %s --input-file %t-hot-count --check-prefix=CHECK-NOPROM --implicit-check-not="[quick_sort:4 @"
+
; CHECK-MAXPROM-DAG: [quick_sort:4 @ t1]:6000:600
; CHECK-MAXPROM-DAG: [quick_sort:4 @ t2]:2000:200
; CHECK-MAXPROM-DAG: [t3]:2000:200
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp
index feb553b6f235f2..ec706157a7694d 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.cpp
+++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp
@@ -25,7 +25,7 @@ STATISTIC(PreInlNumCSNotInlined,
"Number of functions not inlined with context sensitive profile");
STATISTIC(PreInlNumCSICPRejected,
"Number of indirect call target contexts not inlined because the "
- "sample profile loader would not promote them");
+ "pre-inliner ICP checks rejected them");
STATISTIC(PreInlNumCSInlinedHitMinLimit,
"Number of functions with FDO inline stopped due to min size limit");
STATISTIC(PreInlNumCSInlinedHitMaxLimit,
@@ -55,8 +55,8 @@ static cl::opt<bool> SamplePreInlineReplay(
static cl::opt<bool> CSPreinlHonorICPLimits(
"csspgo-preinliner-honor-icp-limits", cl::Hidden, cl::init(true),
- cl::desc("Do not inline indirect call target contexts that the sample "
- "profile loader will not promote (see "
+ cl::desc("Apply indirect-call promotion checks when selecting contexts "
+ "for pre-inlining (see "
"-sample-profile-icp-relative-hotness)."));
static cl::opt<int> CSPreinlMultiplierForPrevInl(
@@ -86,6 +86,19 @@ CSPreInliner::CSPreInliner(SampleContextTracker &Tracker,
ProfileICPRelativeHotnessSkip =
getSampleProfileICPRelativeHotnessSkip(Binary.getLibraryOptions());
ProfileICPMaxProm = getSampleProfileICPMaxProm(Binary.getLibraryOptions());
+
+ // Use the contextless summary computed before pre-inlining. The loader uses
+ // the emitted nested profile, whose summary depends on our inline decisions
+ // and can give a lower hot threshold.
+ //
+ // We deliberately keep the contextless threshold here. Using a lower estimate
+ // could allow more indirect targets to consume the caller's inline size
+ // budget, leaving less room for other profitable callees. Rejecting a context
+ // here leaves it on the existing base-profile merge path, so its samples can
+ // still guide optimization of the out-of-line function. The loader continues
+ // to apply its own hotness check.
+ HotCountThreshold = ProfileSummaryBuilder::getHotCountThreshold(
+ Summary->getDetailedSummary());
}
std::vector<FunctionId> CSPreInliner::buildTopDownOrder() {
@@ -118,15 +131,16 @@ std::vector<FunctionId> CSPreInliner::buildTopDownOrder() {
return Order;
}
-// Find the indirect call targets under CallerNode that the sample loader won't
-// promote, so we don't plan to inline them. The loader (see
+// Find indirect call targets to exclude from the pre-inline plan. Apply the
+// loader's promotion limits, using the contextless hot threshold as described
+// in the constructor. The loader (see
// inlineHotFunctionsWithPriority) walks a call's targets from hottest to
// coldest and promotes one only if it's hot, is at least
// ProfileICPRelativeHotness percent of the site total (once the first
// ProfileICPRelativeHotnessSkip targets are promoted), and fewer than
-// ProfileICPMaxProm targets have been promoted so far. Pre-inlining a target
-// that fails these checks just burns the caller's size budget and leaves its
-// samples nested at the call site instead of in its own profile.
+// ProfileICPMaxProm targets have been promoted so far. Keep rejected targets
+// out of the caller's inline size budget and leave their contexts for merging
+// into their base profiles.
//
// One wrinkle: nesting a target under the caller makes convertCSProfiles drop
// its call-target entry, so the loader counts its head samples instead and the
@@ -157,8 +171,6 @@ CSPreInliner::getICPRejectedCallees(ContextTrieNode *CallerNode,
{CalleeNode, CalleeSamples->getHeadSamplesEstimate()});
}
- uint64_t HotCountThreshold = ProfileSummaryBuilder::getHotCountThreshold(
- Summary->getDetailedSummary());
for (auto &Entry : CallSites) {
const LineLocation &CallSite = Entry.first;
// Only indirect calls are subject to ICP, so check the call probe type. It
@@ -219,7 +231,7 @@ CSPreInliner::getICPRejectedCallees(ContextTrieNode *CallerNode,
for (size_t I = NumAccepted; I < Callees.size(); ++I) {
const Target &T = Callees[I];
Rejected.insert(T.Node);
- LLVM_DEBUG(dbgs() << " Not promotable by ICP: "
+ LLVM_DEBUG(dbgs() << " Rejected by pre-inliner ICP checks: "
<< ContextTracker.getContextString(T.Node) << " (head "
<< T.HeadCount << ", site total " << SiteTotal
<< ")\n");
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.h b/llvm/tools/llvm-profgen/CSPreInliner.h
index 4a587899c87498..de5aeed6ae9f86 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.h
+++ b/llvm/tools/llvm-profgen/CSPreInliner.h
@@ -33,8 +33,8 @@ struct ProfiledInlineCandidate {
uint64_t CallsiteCount;
// Size proxy for function under particular call context.
uint64_t SizeCost;
- // Indirect call target that SampleProfileLoader will not promote, so its
- // context cannot be inlined and must be merged into the callee's base.
+ // Indirect call target rejected by the pre-inliner's ICP checks, so its
+ // context should follow the merge path into the callee's base profile.
bool ICPRejected = false;
};
@@ -92,6 +92,7 @@ class CSPreInliner {
unsigned ProfileICPRelativeHotness;
unsigned ProfileICPRelativeHotnessSkip;
unsigned ProfileICPMaxProm;
+ uint64_t HotCountThreshold;
SampleContextTracker &ContextTracker;
ProfiledBinary &Binary;
ProfileSummary *Summary;
More information about the llvm-branch-commits
mailing list