[llvm] 51ce567 - [SampleProfile] Add all callsites to AllCandidates if InlineReplay is in effect
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 29 12:05:31 PDT 2021
Author: modimo
Date: 2021-10-29T12:04:52-07:00
New Revision: 51ce567b38ec92163ec05c9bef0bd0e2bd53c667
URL: https://github.com/llvm/llvm-project/commit/51ce567b38ec92163ec05c9bef0bd0e2bd53c667
DIFF: https://github.com/llvm/llvm-project/commit/51ce567b38ec92163ec05c9bef0bd0e2bd53c667.diff
LOG: [SampleProfile] Add all callsites to AllCandidates if InlineReplay is in effect
Replay in sample profiling needs to be asked on candidates that may not have counts or below the threshold. If replay is in effect for a function make sure these are captured and also imported during thinLTO.
Testing:
ninja check-all
Reviewed By: wenlei
Differential Revision: https://reviews.llvm.org/D112033
Added:
llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list-no-funca.prof
llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list-replay.txt
llvm/test/Transforms/SampleProfile/Inputs/function_metadata_replay.txt
llvm/test/Transforms/SampleProfile/Inputs/inline-topdown-missing.prof
Modified:
llvm/lib/Transforms/IPO/SampleProfile.cpp
llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list.prof
llvm/test/Transforms/SampleProfile/csspgo-import-list.ll
llvm/test/Transforms/SampleProfile/function_metadata.ll
llvm/test/Transforms/SampleProfile/inline-replay.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index a0b351249d25b..8c4290eb55776 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -404,7 +404,7 @@ class SampleProfileLoader final
findFunctionSamples(const Instruction &I) const override;
std::vector<const FunctionSamples *>
findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
- void findExternalInlineCandidate(const FunctionSamples *Samples,
+ void findExternalInlineCandidate(CallBase *CB, const FunctionSamples *Samples,
DenseSet<GlobalValue::GUID> &InlinedGUIDs,
const StringMap<Function *> &SymbolMap,
uint64_t Threshold);
@@ -412,8 +412,11 @@ class SampleProfileLoader final
bool tryPromoteAndInlineCandidate(
Function &F, InlineCandidate &Candidate, uint64_t SumOrigin,
uint64_t &Sum, SmallVector<CallBase *, 8> *InlinedCallSites = nullptr);
+
bool inlineHotFunctions(Function &F,
DenseSet<GlobalValue::GUID> &InlinedGUIDs);
+ Optional<InlineCost> getExternalInlineAdvisorCost(CallBase &CB);
+ bool getExternalInlineAdvisorShouldInline(CallBase &CB);
InlineCost shouldInlineCandidate(InlineCandidate &Candidate);
bool getInlineCandidate(InlineCandidate *NewCandidate, CallBase *CB);
bool
@@ -977,8 +980,24 @@ void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates(
}
void SampleProfileLoader::findExternalInlineCandidate(
- const FunctionSamples *Samples, DenseSet<GlobalValue::GUID> &InlinedGUIDs,
+ CallBase *CB, const FunctionSamples *Samples,
+ DenseSet<GlobalValue::GUID> &InlinedGUIDs,
const StringMap<Function *> &SymbolMap, uint64_t Threshold) {
+
+ // If ExternalInlineAdvisor wants to inline an external function
+ // make sure it's imported
+ if (CB && getExternalInlineAdvisorShouldInline(*CB)) {
+ // Samples may not exist for replayed function, if so
+ // just add the direct GUID and move on
+ if (!Samples) {
+ InlinedGUIDs.insert(
+ FunctionSamples::getGUID(CB->getCalledFunction()->getName()));
+ return;
+ }
+ // Otherwise, drop the threshold to import everything that we can
+ Threshold = 0;
+ }
+
assert(Samples && "expect non-null caller profile");
// For AutoFDO profile, retrieve candidate profiles by walking over
@@ -1073,16 +1092,20 @@ bool SampleProfileLoader::inlineHotFunctions(
for (auto &I : BB.getInstList()) {
const FunctionSamples *FS = nullptr;
if (auto *CB = dyn_cast<CallBase>(&I)) {
- if (!isa<IntrinsicInst>(I) && (FS = findCalleeFunctionSamples(*CB))) {
- assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
- "GUIDToFuncNameMap has to be populated");
- AllCandidates.push_back(CB);
- if (FS->getEntrySamples() > 0 || ProfileIsCS)
- LocalNotInlinedCallSites.try_emplace(CB, FS);
- if (callsiteIsHot(FS, PSI, ProfAccForSymsInList))
- Hot = true;
- else if (shouldInlineColdCallee(*CB))
- ColdCandidates.push_back(CB);
+ if (!isa<IntrinsicInst>(I)) {
+ if ((FS = findCalleeFunctionSamples(*CB))) {
+ assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
+ "GUIDToFuncNameMap has to be populated");
+ AllCandidates.push_back(CB);
+ if (FS->getEntrySamples() > 0 || ProfileIsCS)
+ LocalNotInlinedCallSites.try_emplace(CB, FS);
+ if (callsiteIsHot(FS, PSI, ProfAccForSymsInList))
+ Hot = true;
+ else if (shouldInlineColdCallee(*CB))
+ ColdCandidates.push_back(CB);
+ } else if (getExternalInlineAdvisorShouldInline(*CB)) {
+ AllCandidates.push_back(CB);
+ }
}
}
}
@@ -1109,7 +1132,7 @@ bool SampleProfileLoader::inlineHotFunctions(
for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
uint64_t SumOrigin = Sum;
if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
- findExternalInlineCandidate(FS, InlinedGUIDs, SymbolMap,
+ findExternalInlineCandidate(I, FS, InlinedGUIDs, SymbolMap,
PSI->getOrCompHotCountThreshold());
continue;
}
@@ -1129,8 +1152,8 @@ bool SampleProfileLoader::inlineHotFunctions(
LocalChanged = true;
}
} else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
- findExternalInlineCandidate(findCalleeFunctionSamples(*I), InlinedGUIDs,
- SymbolMap,
+ findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
+ InlinedGUIDs, SymbolMap,
PSI->getOrCompHotCountThreshold());
}
}
@@ -1259,7 +1282,9 @@ bool SampleProfileLoader::getInlineCandidate(InlineCandidate *NewCandidate,
// Find the callee's profile. For indirect call, find hottest target profile.
const FunctionSamples *CalleeSamples = findCalleeFunctionSamples(*CB);
- if (!CalleeSamples)
+ // If ExternalInlineAdvisor wants to inline this site, do so even
+ // if Samples are not present.
+ if (!CalleeSamples && !getExternalInlineAdvisorShouldInline(*CB))
return false;
float Factor = 1.0;
@@ -1278,11 +1303,11 @@ bool SampleProfileLoader::getInlineCandidate(InlineCandidate *NewCandidate,
return true;
}
-InlineCost
-SampleProfileLoader::shouldInlineCandidate(InlineCandidate &Candidate) {
+Optional<InlineCost>
+SampleProfileLoader::getExternalInlineAdvisorCost(CallBase &CB) {
std::unique_ptr<InlineAdvice> Advice = nullptr;
if (ExternalInlineAdvisor) {
- Advice = ExternalInlineAdvisor->getAdvice(*Candidate.CallInstr);
+ Advice = ExternalInlineAdvisor->getAdvice(CB);
if (Advice) {
if (!Advice->isInliningRecommended()) {
Advice->recordUnattemptedInlining();
@@ -1293,6 +1318,19 @@ SampleProfileLoader::shouldInlineCandidate(InlineCandidate &Candidate) {
}
}
+ return {};
+}
+
+bool SampleProfileLoader::getExternalInlineAdvisorShouldInline(CallBase &CB) {
+ Optional<InlineCost> Cost = getExternalInlineAdvisorCost(CB);
+ return Cost ? !!Cost.getValue() : false;
+}
+
+InlineCost
+SampleProfileLoader::shouldInlineCandidate(InlineCandidate &Candidate) {
+ if (Optional<InlineCost> ReplayCost =
+ getExternalInlineAdvisorCost(*Candidate.CallInstr))
+ return ReplayCost.getValue();
// Adjust threshold based on call site hotness, only do this for callsite
// prioritized inliner because otherwise cost-benefit check is done earlier.
int SampleThreshold = SampleColdCallSiteThreshold;
@@ -1411,7 +1449,7 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
for (const auto *FS : CalleeSamples) {
// TODO: Consider disable pre-lTO ICP for MonoLTO as well
if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
- findExternalInlineCandidate(FS, InlinedGUIDs, SymbolMap,
+ findExternalInlineCandidate(I, FS, InlinedGUIDs, SymbolMap,
PSI->getOrCompHotCountThreshold());
continue;
}
@@ -1459,8 +1497,9 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
Changed = true;
}
} else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
- findExternalInlineCandidate(Candidate.CalleeSamples, InlinedGUIDs,
- SymbolMap, PSI->getOrCompHotCountThreshold());
+ findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
+ InlinedGUIDs, SymbolMap,
+ PSI->getOrCompHotCountThreshold());
}
}
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list-no-funca.prof b/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list-no-funca.prof
new file mode 100644
index 0000000000000..ac8fae4ddddfb
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list-no-funca.prof
@@ -0,0 +1,17 @@
+[main]:154:2
+ 2: 12
+ 3.1: 18 _Z5funcBi:19
+[main:3.1 @ _Z5funcBi]:120:7040
+ 0: 7001
+ 1: 19 _Z8funcLeafi:9999
+ 3: 12
+[main:3.1 @ _Z5funcBi @ _Z5funcBiLeaf2]:1:9010
+ 0: 7001
+ 1: 19 _Z8funcLeafi3:9999
+ 3: 12
+[main:3 @ _Z5funcCi]:23254:11
+ 0: 10
+ 1: 23250
+[main:3 @ _Z5funcDi]:23:45201
+ 0: 10
+ 1: 23250
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list-replay.txt b/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list-replay.txt
new file mode 100644
index 0000000000000..b93149478b175
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list-replay.txt
@@ -0,0 +1,2 @@
+remark: merged.cpp:14:21.1: '_Z5funcBi' inlined into 'main' to match profiling context with (cost=0, threshold=337) at callsite main:3:21.1
+remark: merged.cpp:13:10.1: '_Z5funcAi' inlined into 'main' to match profiling context with (cost=0, threshold=337) at callsite main:2:10.1
\ No newline at end of file
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list.prof b/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list.prof
index c81f8a998b335..f4b7628c55528 100644
--- a/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list.prof
+++ b/llvm/test/Transforms/SampleProfile/Inputs/csspgo-import-list.prof
@@ -1,6 +1,6 @@
[main]:154:2
2: 12
- 3: 18 _Z5funcAi:11
+ 2.1: 18 _Z5funcAi:11
3.1: 18 _Z5funcBi:19
[main:3.1 @ _Z5funcBi]:120:7040
0: 7001
@@ -10,7 +10,7 @@
0: 7001
1: 19 _Z8funcLeafi3:9999
3: 12
-[main:2 @ _Z5funcAi]:99:11
+[main:2.1 @ _Z5funcAi]:99:11
0: 10
1: 10 _Z8funcLeafi:11
2: 287864 _Z3fibi:315608
@@ -21,7 +21,7 @@
[main:3 @ _Z5funcDi]:23:45201
0: 10
1: 23250
-[main:2 @ _Z5funcAi:2 @ _Z3fibi]:120:101
+[main:2.1 @ _Z5funcAi:2 @ _Z3fibi]:120:101
0: 99
1: 6
3: 97
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/function_metadata_replay.txt b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata_replay.txt
new file mode 100644
index 0000000000000..cf528e295d6bd
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata_replay.txt
@@ -0,0 +1 @@
+remark: calls.cc:8:0: 'foo' inlined into 'test_liveness' to match profiling context with (cost=0, threshold=337) at callsite test_liveness:1:0;
\ No newline at end of file
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown-missing.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown-missing.prof
new file mode 100644
index 0000000000000..7a4b4ca3690cc
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown-missing.prof
@@ -0,0 +1,5 @@
+main:225715:0
+ 2.1: 5553
+ 3: 5391
+
+_Z3sumii:6010:50000
\ No newline at end of file
diff --git a/llvm/test/Transforms/SampleProfile/csspgo-import-list.ll b/llvm/test/Transforms/SampleProfile/csspgo-import-list.ll
index cd8ac4188ac45..85cc54c609199 100644
--- a/llvm/test/Transforms/SampleProfile/csspgo-import-list.ll
+++ b/llvm/test/Transforms/SampleProfile/csspgo-import-list.ll
@@ -7,6 +7,13 @@
; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -sample-profile-file=%t.md5 -S | FileCheck %s
; RUN: llvm-profdata show --sample -show-sec-info-only %t.md5 | FileCheck %s --check-prefix=CHECK-ORDERED
+;; Validate that with replay in effect, we import call sites even if they are below the threshold
+;; Baseline import decisions
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -sample-profile-file=%S/Inputs/csspgo-import-list.prof -profile-summary-hot-count=10000 -S | FileCheck %s --check-prefix=THRESHOLD
+;; With replay
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -sample-profile-file=%S/Inputs/csspgo-import-list.prof -sample-profile-inline-replay=%S/Inputs/csspgo-import-list-replay.txt -sample-profile-inline-replay-scope=Module -profile-summary-hot-count=10000 -S | FileCheck %s --check-prefix=THRESHOLD-REPLAY
+;; With replay but no profile information for call to _Z5funcAi. We import _Z5funcAi because it's explicitly in the replay but don't go further to its callee (_Z3fibi) because we lack samples
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -sample-profile-file=%S/Inputs/csspgo-import-list-no-funca.prof -sample-profile-inline-replay=%S/Inputs/csspgo-import-list-replay.txt -sample-profile-inline-replay-scope=Module -profile-summary-hot-count=10000 -S | FileCheck %s --check-prefix=THRESHOLD-REPLAY-NO-FUNCA
declare i32 @_Z5funcBi(i32 %x)
declare i32 @_Z5funcAi(i32 %x)
@@ -32,11 +39,23 @@ for.body: ; preds = %for.body, %entry
}
; Make sure the ImportGUID stays with entry count metadata for ThinLTO-PreLink
+; For reference, the GUIDs of the functions are:
+; _Z5funcBi:446061515086924981
+; _Z8funcLeafi:7102633082150537521
+; _Z5funcBiLeaf2:3815895320998406042
+; _Z8funcLeafi3:-2862076748587597320
+; _Z5funcAi:6309742469962978389
+; _Z3fibi:-2016976694713209516
+
; CHECK: distinct !DISubprogram(name: "main"
; CHECK: !{!"function_entry_count", i64 3, i64 446061515086924981, i64 3815895320998406042, i64 7102633082150537521, i64 -2862076748587597320}
; CHECK-ORDERED: FuncOffsetTableSection {{.*}} {ordered}
+; THRESHOLD: !{!"function_entry_count", i64 3}
+; THRESHOLD-REPLAY: !{!"function_entry_count", i64 3, i64 446061515086924981, i64 3815895320998406042, i64 6309742469962978389, i64 7102633082150537521, i64 -2862076748587597320, i64 -2016976694713209516}
+; THRESHOLD-REPLAY-NO-FUNCA: !{!"function_entry_count", i64 3, i64 446061515086924981, i64 3815895320998406042, i64 6309742469962978389, i64 7102633082150537521, i64 -2862076748587597320}
+
attributes #0 = { nofree noinline norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" "use-sample-profile" }
!llvm.dbg.cu = !{!2}
diff --git a/llvm/test/Transforms/SampleProfile/function_metadata.ll b/llvm/test/Transforms/SampleProfile/function_metadata.ll
index 537ea6da9f556..9ff1e5a9bc884 100644
--- a/llvm/test/Transforms/SampleProfile/function_metadata.ll
+++ b/llvm/test/Transforms/SampleProfile/function_metadata.ll
@@ -2,6 +2,13 @@
; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.compact.afdo -S | FileCheck %s
; RUN: opt < %s -passes='pseudo-probe,thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/pseudo-probe-func-metadata.prof -S | FileCheck %s
+;; Validate that with replay in effect, we import call sites even if they are below the threshold
+;; Baseline import decisions
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-summary-hot-count=2000 -profile-file=%S/Inputs/function_metadata.prof -S | FileCheck -check-prefix=THRESHOLD %s
+;; With replay decisions
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-summary-hot-count=2000 -profile-file=%S/Inputs/function_metadata.prof -S -sample-profile-inline-replay-scope=Function -sample-profile-inline-replay=%S/Inputs/function_metadata_replay.txt | FileCheck -check-prefix=THRESHOLD-REPLAY %s
+
+
; Tests whether the functions in the inline stack are added to the
; function_entry_count metadata.
@@ -16,6 +23,8 @@ define void @bar_available() #0 !dbg !14 {
}
; CHECK: define void @test({{.*}} !prof ![[ENTRY_TEST:[0-9]+]]
+; THRESHOLD: define void @test({{.*}} !prof ![[ENTRY_THRESHOLD:[0-9]+]]
+; THRESHOLD-REPLAY: define void @test({{.*}} !prof ![[ENTRY_REPLAY_TEST:[0-9]+]]
define void @test(void ()*) #0 !dbg !7 {
%2 = alloca void ()*
store void ()* %0, void ()** %2
@@ -27,6 +36,8 @@ define void @test(void ()*) #0 !dbg !7 {
}
; CHECK: define void @test_liveness({{.*}} !prof ![[ENTRY_TEST_LIVENESS:[0-9]+]]
+; THRESHOLD: define void @test_liveness({{.*}} !prof ![[ENTRY_THRESHOLD:[0-9]+]]
+; THRESHOLD-REPLAY: define void @test_liveness({{.*}} !prof ![[ENTRY_REPLAY_TEST_LIVENESS:[0-9]+]]
define void @test_liveness() #0 !dbg !12 {
call void @foo(), !dbg !20
ret void
@@ -43,6 +54,13 @@ define void @test_liveness() #0 !dbg !12 {
; to bar. bar_available should not be included as it's within the same module.
; CHECK: ![[ENTRY_TEST_LIVENESS]] = !{!"function_entry_count", i64 1, i64 6699318081062747564, i64 -2012135647395072713, i64 -1522495160813492905}
+; With high threshold, nothing should be imported
+; THRESHOLD: ![[ENTRY_THRESHOLD]] = !{!"function_entry_count", i64 1}
+
+; With high threshold and replay, sites that are in the replay (foo, and transitively bar and bar_dbg in function test_liveness) should be imported
+; THRESHOLD-REPLAY: ![[ENTRY_REPLAY_TEST]] = !{!"function_entry_count", i64 1}
+; THRESHOLD-REPLAY: ![[ENTRY_REPLAY_TEST_LIVENESS]] = !{!"function_entry_count", i64 1, i64 6699318081062747564, i64 -2012135647395072713, i64 -1522495160813492905}
+
attributes #0 = {"use-sample-profile"}
!llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/SampleProfile/inline-replay.ll b/llvm/test/Transforms/SampleProfile/inline-replay.ll
index 9b640e90ac8ea..828ce87d90e5e 100644
--- a/llvm/test/Transforms/SampleProfile/inline-replay.ll
+++ b/llvm/test/Transforms/SampleProfile/inline-replay.ll
@@ -1,24 +1,36 @@
;; Note that this needs new pass manager for now. Passing `-sample-profile-inline-replay` to legacy pass manager is a no-op.
;; Check baseline inline decisions
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=DEFAULT %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=DEFAULT %s
;; Check replay inline decisions
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=Module -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=Module -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY %s
+
+;; Check baseline inline decisions with all callee counts missing
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-missing.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=DEFAULT-NOINLINING -allow-empty %s
+
+;; Check replay inline decisions with all callee counts missing. The call sites should still be passed to the replay advisor and successfully get inlined as before
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-missing.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=Module -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY %s
+
+;; Check baseline inline decisions with high threshold
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -profile-summary-hot-count=500000 -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=DEFAULT-NOINLINING -allow-empty %s
+
+;; Check replay inline decisions with high threshold. The call sites should still be passed to the replay advisor and successfully get inlined as before
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -profile-summary-hot-count=500000 -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=Module -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY -allow-empty %s
;; Check baseline inline decisions with "inline-topdown-inline-all.prof" which inlines all sites
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=DEFAULT-ALL %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=DEFAULT-ALL %s
;; Check function scope replay inline decisions with "inline-topdown-inline-all.prof" and "inline-topdown-function-scope.txt" which only contains: '_Z3sumii' inlined into 'main'
;; 1. _Z3sumii is inlined into main, but all other inline candidates in main (e.g. _Z3subii) are not inlined
;; 2. Inline decisions made in other functions match default sample inlining, in this case _Z3subii is inlined into _Z3sumii
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-inline-replay=%S/Inputs/inline-replay-function-scope.txt -sample-profile-inline-replay-scope=Function -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ALL-FUNCTION %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-inline-replay=%S/Inputs/inline-replay-function-scope.txt -sample-profile-inline-replay-scope=Function -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ALL-FUNCTION %s
;; Check behavior on non-existent replay file
-; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/non-existent-dummy.txt -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ERROR %s
+; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/non-existent-dummy.txt -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ERROR %s
;; Check scope inlining errors out on non <Module|Function> inputs
-; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=function -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-SCOPE %s
+; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=function -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-SCOPE %s
; DEFAULT: '_Z3sumii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}}
; DEFAULT: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}}
@@ -28,6 +40,8 @@
; REPLAY: '_Z3subii' inlined into 'main' to match profiling context with (cost=always)
; REPLAY-NOT: '_Z3subii' inlined into '_Z3sumii'
+; DEFAULT-NOINLINING-NOT: inlined into
+
; DEFAULT-ALL: '_Z3sumii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}}
; DEFAULT-ALL: '_Z3subii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}}
; DEFAULT-ALL: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}}
More information about the llvm-commits
mailing list