[llvm] [PGO][ICP] Add separate hot count threshold for indirect call promotion (PR #206656)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 29 23:50:36 PDT 2026
https://github.com/flyahead created https://github.com/llvm/llvm-project/pull/206656
Indirect call promotion currently uses the same hot count threshold as the general PGO optimization path (controlled by
-profile-summary-cutoff-hot). This change introduces a dedicated -profile-summary-cutoff-hot-icp option (default 990000 percentile) that allows users to control the ICP promotion threshold independently.
Performance: on MySQL sysbench, setting -profile-summary-cutoff-hot-icp from the default 990000 to 999900 yields a 1.4% improvement.
>From 89d4932602849200e91865a5fd68a3414b3f9466 Mon Sep 17 00:00:00 2001
From: Liu Zhongying <liuzhongying at huawei.com>
Date: Mon, 29 Jun 2026 20:29:15 +0800
Subject: [PATCH] Add separate hot count threshold for indirect call promotion
Indirect call promotion currently uses the same hot count threshold as
the general PGO optimization path (controlled by
-profile-summary-cutoff-hot). This change introduces a dedicated
-profile-summary-cutoff-hot-icp option (default 990000 percentile)
that allows users to control the ICP promotion threshold independently.
This enables more fine-grained control: users can set a higher
percentile to lower the threshold and promote more indirect call
targets, or a lower percentile to raise the threshold and limit code
bloat from over-promotion.
Performance: on MySQL sysbench, setting -profile-summary-cutoff-hot-icp
from the default 990000 to 999900 yields a 1.4% improvement.
---
.../llvm/Analysis/ProfileSummaryInfo.h | 4 +-
llvm/include/llvm/ProfileData/ProfileCommon.h | 3 +
llvm/lib/Analysis/ProfileSummaryInfo.cpp | 7 +-
.../lib/ProfileData/ProfileSummaryBuilder.cpp | 15 ++++
.../Instrumentation/IndirectCallPromotion.cpp | 2 +-
.../PGOProfile/icp-hot-threshold.ll | 81 +++++++++++++++++++
6 files changed, 108 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/Transforms/PGOProfile/icp-hot-threshold.ll
diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
index 55b195b93f6da..a9c1c595e8073 100644
--- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -46,6 +46,8 @@ class ProfileSummaryInfo {
void computeThresholds();
// Count thresholds to answer isHotCount and isColdCount queries.
std::optional<uint64_t> HotCountThreshold, ColdCountThreshold;
+ // Count threshold for indirect call promotion hotness.
+ std::optional<uint64_t> HotCountThresholdForICP;
// True if the working set size of the code is considered huge,
// because the number of profile counts required to reach the hot
// percentile is above a huge threshold.
@@ -179,7 +181,7 @@ class ProfileSummaryInfo {
PercentileCutoff, F, BFI);
}
/// Returns true if count \p C is considered hot.
- LLVM_ABI bool isHotCount(uint64_t C) const;
+ LLVM_ABI bool isHotCount(uint64_t C, bool isForICP = false) const;
/// Returns true if count \p C is considered cold.
LLVM_ABI bool isColdCount(uint64_t C) const;
/// Returns true if count \p C is considered hot with regard to a given
diff --git a/llvm/include/llvm/ProfileData/ProfileCommon.h b/llvm/include/llvm/ProfileData/ProfileCommon.h
index 6f71097a94a43..3066617467963 100644
--- a/llvm/include/llvm/ProfileData/ProfileCommon.h
+++ b/llvm/include/llvm/ProfileData/ProfileCommon.h
@@ -31,6 +31,7 @@ namespace llvm {
LLVM_ABI extern cl::opt<bool> UseContextLessSummary;
LLVM_ABI extern cl::opt<int> ProfileSummaryCutoffHot;
+LLVM_ABI extern cl::opt<int> ProfileSummaryCutoffHotICP;
LLVM_ABI extern cl::opt<int> ProfileSummaryCutoffCold;
LLVM_ABI extern cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold;
LLVM_ABI extern cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold;
@@ -74,6 +75,8 @@ class ProfileSummaryBuilder {
LLVM_ABI static const ProfileSummaryEntry &
getEntryForPercentile(const SummaryEntryVector &DS, uint64_t Percentile);
LLVM_ABI static uint64_t getHotCountThreshold(const SummaryEntryVector &DS);
+ LLVM_ABI static uint64_t
+ getHotCountThresholdForICP(const SummaryEntryVector &DS);
LLVM_ABI static uint64_t getColdCountThreshold(const SummaryEntryVector &DS);
};
diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
index a50c34d1b1007..7bf131913a5e8 100644
--- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp
+++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
@@ -126,6 +126,8 @@ void ProfileSummaryInfo::computeThresholds() {
ProfileSummaryBuilder::getHotCountThreshold(DetailedSummary);
ColdCountThreshold =
ProfileSummaryBuilder::getColdCountThreshold(DetailedSummary);
+ HotCountThresholdForICP =
+ ProfileSummaryBuilder::getHotCountThresholdForICP(DetailedSummary);
// When the hot and cold thresholds are identical, we would classify
// a count value as both hot and cold since we are doing an inclusive check
// (see ::is{Hot|Cold}Count(). To avoid this undesirable overlap, ensure the
@@ -180,8 +182,9 @@ bool ProfileSummaryInfo::hasLargeWorkingSetSize() const {
return HasLargeWorkingSetSize && *HasLargeWorkingSetSize;
}
-bool ProfileSummaryInfo::isHotCount(uint64_t C) const {
- return HotCountThreshold && C >= *HotCountThreshold;
+bool ProfileSummaryInfo::isHotCount(uint64_t C, bool isForICP) const {
+ auto &Threshold = isForICP ? HotCountThresholdForICP : HotCountThreshold;
+ return Threshold && C >= *Threshold;
}
bool ProfileSummaryInfo::isColdCount(uint64_t C) const {
diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
index 23c87702eb133..cf846116a3814 100644
--- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
+++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
@@ -35,6 +35,11 @@ cl::opt<int> ProfileSummaryCutoffHot(
cl::desc("A count is hot if it exceeds the minimum count to"
" reach this percentile of total counts."));
+cl::opt<int> ProfileSummaryCutoffHotICP(
+ "profile-summary-cutoff-hot-icp", cl::Hidden, cl::init(990000),
+ cl::desc("A count is hot for indirect call promotion if it exceeds"
+ " the minimum count to reach this percentile of total counts."));
+
cl::opt<int> ProfileSummaryCutoffCold(
"profile-summary-cutoff-cold", cl::Hidden, cl::init(999999),
cl::desc("A count is cold if it is below the minimum count"
@@ -181,6 +186,16 @@ ProfileSummaryBuilder::getHotCountThreshold(const SummaryEntryVector &DS) {
return HotCountThreshold;
}
+uint64_t ProfileSummaryBuilder::getHotCountThresholdForICP(
+ const SummaryEntryVector &DS) {
+ auto &HotEntry = ProfileSummaryBuilder::getEntryForPercentile(
+ DS, ProfileSummaryCutoffHotICP);
+ uint64_t HotCountThreshold = HotEntry.MinCount;
+ if (ProfileSummaryHotCount.getNumOccurrences() > 0)
+ HotCountThreshold = ProfileSummaryHotCount;
+ return HotCountThreshold;
+}
+
uint64_t
ProfileSummaryBuilder::getColdCountThreshold(const SummaryEntryVector &DS) {
auto &ColdEntry = ProfileSummaryBuilder::getEntryForPercentile(
diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index cc6361f03d89b..c82775d9e5b42 100644
--- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -882,7 +882,7 @@ bool IndirectCallPromoter::processFunction(ProfileSummaryInfo *PSI) {
continue;
}
// Only pormote hot if ICPAllowHotOnly is true.
- if (ICPAllowHotOnly && !PSI->isHotCount(TotalCount)) {
+ if (ICPAllowHotOnly && !PSI->isHotCount(TotalCount, true)) {
LLVM_DEBUG(dbgs() << "Don't promote the non-hot candidate: TotalCount="
<< TotalCount << "\n");
continue;
diff --git a/llvm/test/Transforms/PGOProfile/icp-hot-threshold.ll b/llvm/test/Transforms/PGOProfile/icp-hot-threshold.ll
new file mode 100644
index 0000000000000..7627cbacc6f48
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/icp-hot-threshold.ll
@@ -0,0 +1,81 @@
+; RUN: opt < %s -passes=pgo-icall-prom -profile-summary-cutoff-hot-icp=200000 -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
+; RUN: opt < %s -passes=pgo-icall-prom -profile-summary-cutoff-hot-icp=100000 -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=FAIL-REMARK
+
+; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func4 with count 5 out of 14
+; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func2 with count 4 out of 9
+; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func3 with count 3 out of 5
+
+; FAIL-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func4
+; FAIL-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func2
+; FAIL-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func3
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at foo = common global ptr null, align 8
+
+define i32 @func1() {
+entry:
+ ret i32 0
+}
+
+define i32 @func2() {
+entry:
+ ret i32 1
+}
+
+define i32 @func3() {
+entry:
+ ret i32 2
+}
+
+define i32 @func4() {
+entry:
+ ret i32 3
+}
+
+define i32 @bar() {
+entry:
+ %tmp = load ptr, ptr @foo, align 8
+ %call = call i32 %tmp(), !prof !34
+ ret i32 %call
+}
+
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{i32 7, !"frame-pointer", i32 1}
+!5 = !{i32 1, !"ProfileSummary", !6}
+!6 = !{!7, !8, !9, !10, !11, !12, !13, !14, !15, !16}
+!7 = !{!"ProfileFormat", !"InstrProf"}
+!8 = !{!"TotalCount", i64 3}
+!9 = !{!"MaxCount", i64 1}
+!10 = !{!"MaxInternalCount", i64 1}
+!11 = !{!"MaxFunctionCount", i64 1}
+!12 = !{!"NumCounts", i64 7}
+!13 = !{!"NumFunctions", i64 4}
+!14 = !{!"IsPartialProfile", i64 0}
+!15 = !{!"PartialProfileRatio", double 0.000000e+00}
+!16 = !{!"DetailedSummary", !17}
+!17 = !{!18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30, !31, !32, !33}
+!18 = !{i32 10000, i64 16, i32 1}
+!19 = !{i32 100000, i64 15, i32 2}
+!20 = !{i32 200000, i64 14, i32 3}
+!21 = !{i32 300000, i64 13, i32 4}
+!22 = !{i32 400000, i64 12, i32 5}
+!23 = !{i32 500000, i64 11, i32 6}
+!24 = !{i32 600000, i64 10, i32 7}
+!25 = !{i32 700000, i64 9, i32 8}
+!26 = !{i32 800000, i64 8, i32 9}
+!27 = !{i32 900000, i64 7, i32 10}
+!28 = !{i32 950000, i64 6, i32 11}
+!29 = !{i32 990000, i64 5, i32 12}
+!30 = !{i32 999000, i64 4, i32 13}
+!31 = !{i32 999900, i64 3, i32 14}
+!32 = !{i32 999990, i64 2, i32 15}
+!33 = !{i32 999999, i64 1, i32 16}
+!34 = !{!"VP", i32 0, i64 14, i64 7651369219802541373, i64 5, i64 -4377547752858689819, i64 4, i64 -6929281286627296573, i64 3, i64 -2545542355363006406, i64 2}
More information about the llvm-commits
mailing list