[llvm] [SampleProfile] Ignore unsampled callees in profile anchors (PR #219335)
Kunal Pathak via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 31 17:02:00 PDT 2026
https://github.com/kunalspathak updated https://github.com/llvm/llvm-project/pull/219335
>From b24c45bb6b0b2002627625fc9d0262f6106cbf38 Mon Sep 17 00:00:00 2001
From: Kunal Pathak <kupathak at meta.com>
Date: Sat, 15 Aug 2026 09:49:11 -0700
Subject: [PATCH 1/2] [SampleProfile] Ignore unsampled callees in profile
anchors
Zero-count inline frames can survive profile flattening as body call targets. Counting them as additional callees converts a uniquely sampled direct callee into UnknownIndirectCallee and causes its samples to be discarded as stale.
Ignore zero-count entries when the same location has a sampled callee, while preserving multiple sampled callees and all-zero locations. Cover both flattened call targets and unflattened callsite samples.
---
.../Transforms/IPO/SampleProfileMatcher.cpp | 21 +++++++-
.../profile-mismatch-indirect-call.prof | 4 ++
...ile-mismatch-multiple-sampled-callees.prof | 19 +++++++
...do-probe-profile-mismatch-zero-callee.prof | 19 +++++++
.../profile-mismatch-indirect-call.ll | 51 +++++++++++++++++++
.../pseudo-probe-profile-mismatch.ll | 9 ++++
6 files changed, 121 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/profile-mismatch-indirect-call.prof
create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile-mismatch-multiple-sampled-callees.prof
create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile-mismatch-zero-callee.prof
create mode 100644 llvm/test/Transforms/SampleProfile/profile-mismatch-indirect-call.ll
diff --git a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
index ffd6a265dafc8..c2974b05b5335 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/IPO/SampleProfileMatcher.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Demangle/Demangle.h"
#include "llvm/IR/IntrinsicInst.h"
@@ -158,16 +159,32 @@ void SampleProfileMatcher::findProfileAnchors(const FunctionSamples &FS,
const LineLocation &Loc = I.first;
if (isInvalidLineOffset(Loc.LineOffset))
continue;
- for (const auto &C : I.second.getCallTargets())
+
+ const auto &CallTargets = I.second.getCallTargets();
+ const bool HasSampledTarget =
+ llvm::any_of(CallTargets, [](const auto &C) { return C.second != 0; });
+ for (const auto &C : CallTargets) {
+ // Zero-count targets carry no evidence of another sampled callee.
+ if (HasSampledTarget && C.second == 0)
+ continue;
InsertAnchor(Loc, C.first, ProfileAnchors);
+ }
}
for (const auto &I : FS.getCallsiteSamples()) {
const LineLocation &Loc = I.first;
if (isInvalidLineOffset(Loc.LineOffset))
continue;
- for (const auto &C : I.second)
+
+ const auto &Callees = I.second;
+ const bool HasSampledCallee = llvm::any_of(
+ Callees, [](const auto &C) { return C.second.getTotalSamples() != 0; });
+ for (const auto &C : Callees) {
+ // Zero-sample inline frames carry no evidence of another call target.
+ if (HasSampledCallee && C.second.getTotalSamples() == 0)
+ continue;
InsertAnchor(Loc, C.first, ProfileAnchors);
+ }
}
}
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/profile-mismatch-indirect-call.prof b/llvm/test/Transforms/SampleProfile/Inputs/profile-mismatch-indirect-call.prof
new file mode 100644
index 0000000000000..4a13ae30f6f41
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/profile-mismatch-indirect-call.prof
@@ -0,0 +1,4 @@
+test:300:0
+ 1: 100 foo:100 unsampled:0
+ 2: 100 bar:100
+ 3: 100 qux:100
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile-mismatch-multiple-sampled-callees.prof b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile-mismatch-multiple-sampled-callees.prof
new file mode 100644
index 0000000000000..851020486e9b1
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile-mismatch-multiple-sampled-callees.prof
@@ -0,0 +1,19 @@
+main:31:0
+ 1: 0
+ 12: matched:10
+ 1: 10
+ !CFGChecksum: 4294967295
+ 12: cleanup:1
+ 1: 1
+ !CFGChecksum: 4294967295
+ 20: 10 bar:10
+ 13: foo_mismatch:10
+ 1: 10
+ !CFGChecksum: 4294967295
+ !CFGChecksum: 84463533171543
+bar:10:10
+ 1: 10
+ !CFGChecksum: 42949671295
+matched:10:10
+ 1: 10
+ !CFGChecksum: 4294967295
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile-mismatch-zero-callee.prof b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile-mismatch-zero-callee.prof
new file mode 100644
index 0000000000000..378bfc64bf6de
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile-mismatch-zero-callee.prof
@@ -0,0 +1,19 @@
+main:30:0
+ 1: 0
+ 12: matched:10
+ 1: 10
+ !CFGChecksum: 4294967295
+ 12: cleanup:0
+ 1: 0
+ !CFGChecksum: 4294967295
+ 20: 10 bar:10
+ 13: foo_mismatch:10
+ 1: 10
+ !CFGChecksum: 4294967295
+ !CFGChecksum: 84463533171543
+bar:10:10
+ 1: 10
+ !CFGChecksum: 42949671295
+matched:10:10
+ 1: 10
+ !CFGChecksum: 4294967295
diff --git a/llvm/test/Transforms/SampleProfile/profile-mismatch-indirect-call.ll b/llvm/test/Transforms/SampleProfile/profile-mismatch-indirect-call.ll
new file mode 100644
index 0000000000000..a984236d21393
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/profile-mismatch-indirect-call.ll
@@ -0,0 +1,51 @@
+; REQUIRES: x86_64-linux
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-mismatch-indirect-call.prof -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll
+; RUN: FileCheck %s --input-file %t
+; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD
+
+; The profile records one sampled target, "foo", and one unsampled target at the
+; location of the IR indirect call(line offset 1). This is not a mismatch since
+; only one call target was sampled. Only the "baz"(IR) vs "qux"(profile)
+; callsite at line offset 3 is a real mismatch.
+
+; CHECK: (1/3) of callsites' profile are invalid and (100/300) of samples are discarded due to callsite location mismatch.
+
+; CHECK-MD: ![[#]] = !{!"NumMismatchedCallsites", i64 1, !"NumRecoveredCallsites", i64 0, !"TotalProfiledCallsites", i64 3, !"MismatchedCallsiteSamples", i64 100, !"RecoveredCallsiteSamples", i64 0}
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at fp = dso_local local_unnamed_addr global ptr null, align 8
+
+define dso_local void @test() local_unnamed_addr #0 !dbg !9 {
+entry:
+ %0 = load ptr, ptr @fp, align 8, !dbg !12
+ tail call void %0(), !dbg !12
+ tail call void @bar(), !dbg !13
+ tail call void @baz(), !dbg !14
+ ret void, !dbg !15
+}
+
+declare void @bar() local_unnamed_addr
+
+declare void @baz() local_unnamed_addr
+
+attributes #0 = { nounwind uwtable "use-sample-profile" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, debugInfoForProfiling: true, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "test")
+!3 = !{i32 7, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 7, !"uwtable", i32 2}
+!6 = !{!"clang"}
+!9 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 5, type: !10, scopeLine: 5, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!10 = !DISubroutineType(types: !11)
+!11 = !{null}
+!12 = !DILocation(line: 6, column: 3, scope: !9)
+!13 = !DILocation(line: 7, column: 3, scope: !9)
+!14 = !DILocation(line: 8, column: 3, scope: !9)
+!15 = !DILocation(line: 9, column: 1, scope: !9)
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
index e1d717c63e9ed..6782fbd60c27b 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
@@ -7,6 +7,8 @@
; RUN: llc < %t.ll -filetype=asm -o - | FileCheck %s --check-prefix=CHECK-ASM
; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch-nested.prof --salvage-unused-profile=false -report-profile-staleness -persist-profile-staleness -S 2>&1 | FileCheck %s --check-prefix=CHECK-NESTED
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch-zero-callee.prof --salvage-unused-profile=false -report-profile-staleness -S 2>&1 | FileCheck %s --check-prefix=CHECK-ZERO-CALLEE
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch-multiple-sampled-callees.prof --salvage-unused-profile=false -report-profile-staleness -S 2>&1 | FileCheck %s --check-prefix=CHECK-MULTIPLE-SAMPLED
; CHECK: (2/3) of functions' profile are invalid and (40/50) of samples are discarded due to function hash mismatch.
@@ -59,6 +61,13 @@
; CHECK-NESTED: (1/2) of functions' profile are invalid and (211/311) of samples are discarded due to function hash mismatch.
+; A zero-sample inline frame at the same probe does not turn the uniquely
+; sampled direct callee into an indirect-call anchor.
+; CHECK-ZERO-CALLEE: (2/3) of callsites' profile are invalid and (20/50) of samples are discarded due to callsite location mismatch.
+
+; Multiple sampled inline callees at one probe remain an indirect-call anchor.
+; CHECK-MULTIPLE-SAMPLED: (3/3) of callsites' profile are invalid and (31/51) of samples are discarded due to callsite location mismatch.
+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
>From 4348fbae2b860506c7997de54cee4b678b55a134 Mon Sep 17 00:00:00 2001
From: Kunal Pathak <kupathak at meta.com>
Date: Mon, 31 Aug 2026 16:20:36 -0700
Subject: [PATCH 2/2] Fix for indirect callsite
After zero-count targets are removed, an indirect callsite profile may
contain only one sampled callee and therefore appear direct.
Treat UnknownIndirectCallee on the IR side as compatible with any profile
callee. Keep the comparison asymmetric so a direct IR call still does not
match a profile containing multiple sampled callees.
---
.../Transforms/IPO/SampleProfileMatcher.h | 2 ++
.../Transforms/IPO/SampleProfileMatcher.cpp | 13 ++++++++-
.../profile-mismatch-indirect-call.prof | 7 +++--
.../profile-mismatch-indirect-call.ll | 29 +++++++++++--------
4 files changed, 35 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileMatcher.h b/llvm/include/llvm/Transforms/IPO/SampleProfileMatcher.h
index 2ae83dab5bcfd..1cce006e260ab 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleProfileMatcher.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleProfileMatcher.h
@@ -156,6 +156,8 @@ class SampleProfileMatcher {
void findIRAnchors(const Function &F, AnchorMap &IRAnchors) const;
void findProfileAnchors(const FunctionSamples &FS,
AnchorMap &ProfileAnchors) const;
+ bool anchorsMatch(const FunctionId &IRAnchor,
+ const FunctionId &ProfileAnchor) const;
// Record the callsite match states for profile staleness report, the result
// is saved in FuncCallsiteMatchStates.
void recordCallsiteMatchStates(const Function &F, const AnchorMap &IRAnchors,
diff --git a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
index c2974b05b5335..b1a65c3c18fa2 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
@@ -188,6 +188,15 @@ void SampleProfileMatcher::findProfileAnchors(const FunctionSamples &FS,
}
}
+bool SampleProfileMatcher::anchorsMatch(const FunctionId &IRAnchor,
+ const FunctionId &ProfileAnchor) const {
+ // An indirect IR call has no statically known callee, so any profiled
+ // target is compatible. The reverse is not true: multiple sampled profile
+ // targets cannot match a direct IR call.
+ return IRAnchor == ProfileAnchor ||
+ IRAnchor == FunctionId(UnknownIndirectCallee);
+}
+
bool SampleProfileMatcher::functionHasProfile(const FunctionId &IRFuncName,
Function *&FuncWithoutProfile) {
FuncWithoutProfile = nullptr;
@@ -236,6 +245,8 @@ SampleProfileMatcher::longestCommonSequence(const AnchorList &AnchorList1,
llvm::longestCommonSequence<LineLocation, FunctionId>(
AnchorList1, AnchorList2,
[&](const FunctionId &A, const FunctionId &B) {
+ if (anchorsMatch(A, B))
+ return true;
return functionMatchesProfile(
A, B,
!MatchUnusedFunction // Find matched function only
@@ -540,7 +551,7 @@ void SampleProfileMatcher::recordCallsiteMatchStates(
if (It == ProfileAnchors.end())
continue;
const auto &ProfCalleeId = It->second;
- if (IRCalleeId == ProfCalleeId) {
+ if (anchorsMatch(IRCalleeId, ProfCalleeId)) {
auto It = CallsiteMatchStates.find(ProfileLoc);
if (It == CallsiteMatchStates.end())
CallsiteMatchStates.try_emplace(ProfileLoc, MatchState::InitialMatch);
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/profile-mismatch-indirect-call.prof b/llvm/test/Transforms/SampleProfile/Inputs/profile-mismatch-indirect-call.prof
index 4a13ae30f6f41..8ae0bd1c66c3f 100644
--- a/llvm/test/Transforms/SampleProfile/Inputs/profile-mismatch-indirect-call.prof
+++ b/llvm/test/Transforms/SampleProfile/Inputs/profile-mismatch-indirect-call.prof
@@ -1,4 +1,5 @@
-test:300:0
+test:400:0
1: 100 foo:100 unsampled:0
- 2: 100 bar:100
- 3: 100 qux:100
+ 2: 100 foo:100 unsampled:0
+ 3: 100 bar:100
+ 4: 100 qux:100
diff --git a/llvm/test/Transforms/SampleProfile/profile-mismatch-indirect-call.ll b/llvm/test/Transforms/SampleProfile/profile-mismatch-indirect-call.ll
index a984236d21393..13dcf0ae51e18 100644
--- a/llvm/test/Transforms/SampleProfile/profile-mismatch-indirect-call.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-mismatch-indirect-call.ll
@@ -3,14 +3,15 @@
; RUN: FileCheck %s --input-file %t
; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD
-; The profile records one sampled target, "foo", and one unsampled target at the
-; location of the IR indirect call(line offset 1). This is not a mismatch since
-; only one call target was sampled. Only the "baz"(IR) vs "qux"(profile)
-; callsite at line offset 3 is a real mismatch.
+; The profile records one sampled target, "foo", and one unsampled target at
+; both a direct call to "foo" and an indirect call. The direct call matches the
+; sampled target, while the indirect IR call is compatible with any profiled
+; target. Only the "baz" (IR) vs "qux" (profile) callsite at line offset 4 is
+; a real mismatch.
-; CHECK: (1/3) of callsites' profile are invalid and (100/300) of samples are discarded due to callsite location mismatch.
+; CHECK: (1/4) of callsites' profile are invalid and (100/400) of samples are discarded due to callsite location mismatch.
-; CHECK-MD: ![[#]] = !{!"NumMismatchedCallsites", i64 1, !"NumRecoveredCallsites", i64 0, !"TotalProfiledCallsites", i64 3, !"MismatchedCallsiteSamples", i64 100, !"RecoveredCallsiteSamples", i64 0}
+; CHECK-MD: ![[#]] = !{!"NumMismatchedCallsites", i64 1, !"NumRecoveredCallsites", i64 0, !"TotalProfiledCallsites", i64 4, !"MismatchedCallsiteSamples", i64 100, !"RecoveredCallsiteSamples", i64 0}
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
@@ -19,13 +20,16 @@ target triple = "x86_64-unknown-linux-gnu"
define dso_local void @test() local_unnamed_addr #0 !dbg !9 {
entry:
- %0 = load ptr, ptr @fp, align 8, !dbg !12
- tail call void %0(), !dbg !12
- tail call void @bar(), !dbg !13
- tail call void @baz(), !dbg !14
- ret void, !dbg !15
+ tail call void @foo(), !dbg !12
+ %0 = load ptr, ptr @fp, align 8, !dbg !13
+ tail call void %0(), !dbg !13
+ tail call void @bar(), !dbg !14
+ tail call void @baz(), !dbg !15
+ ret void, !dbg !16
}
+declare void @foo() local_unnamed_addr
+
declare void @bar() local_unnamed_addr
declare void @baz() local_unnamed_addr
@@ -48,4 +52,5 @@ attributes #0 = { nounwind uwtable "use-sample-profile" }
!12 = !DILocation(line: 6, column: 3, scope: !9)
!13 = !DILocation(line: 7, column: 3, scope: !9)
!14 = !DILocation(line: 8, column: 3, scope: !9)
-!15 = !DILocation(line: 9, column: 1, scope: !9)
+!15 = !DILocation(line: 9, column: 3, scope: !9)
+!16 = !DILocation(line: 10, column: 1, scope: !9)
More information about the llvm-commits
mailing list