[llvm] 7e99bdd - [CSSPGO] Support of CS profiles in extended binary format.
Hongtao Yu via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 27 21:30:23 PST 2021
Author: Hongtao Yu
Date: 2021-01-27T21:29:46-08:00
New Revision: 7e99bddfeaab2713a8bb6ca538da25b66e6efc59
URL: https://github.com/llvm/llvm-project/commit/7e99bddfeaab2713a8bb6ca538da25b66e6efc59
DIFF: https://github.com/llvm/llvm-project/commit/7e99bddfeaab2713a8bb6ca538da25b66e6efc59.diff
LOG: [CSSPGO] Support of CS profiles in extended binary format.
This change brings up support of context-sensitive profiles in the format of extended binary. Existing sample profile reader/writer/merger code is being tweaked to reflect the fact of bracketed input contexts, like (`[...]`). The paired brackets are also needed in extbinary profiles because we don't yet have an otherwise good way to tell calling contexts apart from regular function names since the context delimiter `@` can somehow serve as a part of the C++ mangled names.
Reviewed By: wmi, wenlei
Differential Revision: https://reviews.llvm.org/D95547
Added:
llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext
llvm/test/tools/llvm-profdata/cs-sample-profile.test
Modified:
llvm/include/llvm/ProfileData/SampleProf.h
llvm/include/llvm/ProfileData/SampleProfReader.h
llvm/lib/ProfileData/SampleProfReader.cpp
llvm/lib/ProfileData/SampleProfWriter.cpp
llvm/lib/Transforms/IPO/SampleContextTracker.cpp
llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
llvm/tools/llvm-profdata/llvm-profdata.cpp
llvm/tools/llvm-profgen/ProfileGenerator.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index c45ace9e68c1..346bc4c81d86 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -439,9 +439,11 @@ class SampleContext {
void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
bool hasContext() const { return State != UnknownContext; }
bool isBaseContext() const { return CallingContext.empty(); }
- StringRef getName() const { return Name; }
+ StringRef getNameWithoutContext() const { return Name; }
StringRef getCallingContext() const { return CallingContext; }
- StringRef getNameWithContext() const { return FullContext; }
+ StringRef getNameWithContext(bool WithBracket = false) const {
+ return WithBracket ? InputContext : FullContext;
+ }
private:
// Give a context string, decode and populate internal states like
@@ -449,6 +451,7 @@ class SampleContext {
// `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
void setContext(StringRef ContextStr, ContextStateMask CState) {
assert(!ContextStr.empty());
+ InputContext = ContextStr;
// Note that `[]` wrapped input indicates a full context string, otherwise
// it's treated as context-less function name only.
bool HasContext = ContextStr.startswith("[");
@@ -480,6 +483,9 @@ class SampleContext {
}
}
+ // Input context string including bracketed calling context and leaf function
+ // name
+ StringRef InputContext;
// Full context string including calling context and leaf function name
StringRef FullContext;
// Function name for the associated sample profile
@@ -676,7 +682,8 @@ class FunctionSamples {
Name = Other.getName();
if (!GUIDToFuncNameMap)
GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
-
+ if (Context.getNameWithContext(true).empty())
+ Context = Other.getContext();
if (FunctionHash == 0) {
// Set the function hash code for the target profile.
FunctionHash = Other.getFunctionHash();
@@ -743,8 +750,10 @@ class FunctionSamples {
StringRef getName() const { return Name; }
/// Return function name with context.
- StringRef getNameWithContext() const {
- return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name;
+ StringRef getNameWithContext(bool WithBracket = false) const {
+ return FunctionSamples::ProfileIsCS
+ ? Context.getNameWithContext(WithBracket)
+ : Name;
}
/// Return the original function name.
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index 3f52a2f6163b..999e75eddffa 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -488,8 +488,12 @@ class SampleProfileReader {
/// \brief Whether samples are collected based on pseudo probes.
bool ProfileIsProbeBased = false;
+ /// Whether function profiles are context-sensitive.
bool ProfileIsCS = false;
+ /// Number of context-sensitive profiles.
+ uint32_t CSProfileCount = 0;
+
/// \brief The format of sample.
SampleProfileFormat Format = SPF_None;
};
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index c42931174bc0..c9f41687c356 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -222,8 +222,6 @@ std::error_code SampleProfileReaderText::readImpl() {
sampleprof_error Result = sampleprof_error::success;
InlineCallStack InlineStack;
- int CSProfileCount = 0;
- int RegularProfileCount = 0;
uint32_t ProbeProfileCount = 0;
// SeenMetadata tracks whether we have processed metadata for the current
@@ -257,11 +255,9 @@ std::error_code SampleProfileReaderText::readImpl() {
SampleContext FContext(FName);
if (FContext.hasContext())
++CSProfileCount;
- else
- ++RegularProfileCount;
Profiles[FContext] = FunctionSamples();
FunctionSamples &FProfile = Profiles[FContext];
- FProfile.setName(FContext.getName());
+ FProfile.setName(FContext.getNameWithoutContext());
FProfile.setContext(FContext);
MergeResult(Result, FProfile.addTotalSamples(NumSamples));
MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
@@ -324,13 +320,14 @@ std::error_code SampleProfileReaderText::readImpl() {
}
}
- assert((RegularProfileCount == 0 || CSProfileCount == 0) &&
+ assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
"Cannot have both context-sensitive and regular profile");
ProfileIsCS = (CSProfileCount > 0);
assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) &&
"Cannot have both probe-based profiles and regular profiles");
ProfileIsProbeBased = (ProbeProfileCount > 0);
FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
+ FunctionSamples::ProfileIsCS = ProfileIsCS;
if (Result == sampleprof_error::success)
computeSummary();
@@ -546,12 +543,16 @@ SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
if (std::error_code EC = FName.getError())
return EC;
- Profiles[*FName] = FunctionSamples();
- FunctionSamples &FProfile = Profiles[*FName];
- FProfile.setName(*FName);
-
+ SampleContext FContext(*FName);
+ Profiles[FContext] = FunctionSamples();
+ FunctionSamples &FProfile = Profiles[FContext];
+ FProfile.setName(FContext.getNameWithoutContext());
+ FProfile.setContext(FContext);
FProfile.addHeadSamples(*NumHeadSamples);
+ if (FContext.hasContext())
+ CSProfileCount++;
+
if (std::error_code EC = readProfile(FProfile))
return EC;
return sampleprof_error::success;
@@ -654,40 +655,44 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
return EC;
}
assert(Data == End && "More data is read than expected");
- return sampleprof_error::success;
- }
-
- if (Remapper) {
- for (auto Name : FuncsToUse) {
- Remapper->insert(Name);
+ } else {
+ if (Remapper) {
+ for (auto Name : FuncsToUse) {
+ Remapper->insert(Name);
+ }
}
- }
- if (useMD5()) {
- for (auto Name : FuncsToUse) {
- auto GUID = std::to_string(MD5Hash(Name));
- auto iter = FuncOffsetTable.find(StringRef(GUID));
- if (iter == FuncOffsetTable.end())
- continue;
- const uint8_t *FuncProfileAddr = Start + iter->second;
- assert(FuncProfileAddr < End && "out of LBRProfile section");
- if (std::error_code EC = readFuncProfile(FuncProfileAddr))
- return EC;
- }
- } else {
- for (auto NameOffset : FuncOffsetTable) {
- auto FuncName = NameOffset.first;
- if (!FuncsToUse.count(FuncName) &&
- (!Remapper || !Remapper->exist(FuncName)))
- continue;
- const uint8_t *FuncProfileAddr = Start + NameOffset.second;
- assert(FuncProfileAddr < End && "out of LBRProfile section");
- if (std::error_code EC = readFuncProfile(FuncProfileAddr))
- return EC;
+ if (useMD5()) {
+ for (auto Name : FuncsToUse) {
+ auto GUID = std::to_string(MD5Hash(Name));
+ auto iter = FuncOffsetTable.find(StringRef(GUID));
+ if (iter == FuncOffsetTable.end())
+ continue;
+ const uint8_t *FuncProfileAddr = Start + iter->second;
+ assert(FuncProfileAddr < End && "out of LBRProfile section");
+ if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+ return EC;
+ }
+ } else {
+ for (auto NameOffset : FuncOffsetTable) {
+ SampleContext FContext(NameOffset.first);
+ auto FuncName = FContext.getNameWithoutContext();
+ if (!FuncsToUse.count(FuncName) &&
+ (!Remapper || !Remapper->exist(FuncName)))
+ continue;
+ const uint8_t *FuncProfileAddr = Start + NameOffset.second;
+ assert(FuncProfileAddr < End && "out of LBRProfile section");
+ if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+ return EC;
+ }
}
+ Data = End;
}
- Data = End;
+ assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
+ "Cannot have both context-sensitive and regular profile");
+ ProfileIsCS = (CSProfileCount > 0);
+ FunctionSamples::ProfileIsCS = ProfileIsCS;
return sampleprof_error::success;
}
@@ -887,7 +892,8 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
if (std::error_code EC = Checksum.getError())
return EC;
- Profiles[*FName].setFunctionHash(*Checksum);
+ SampleContext FContext(*FName);
+ Profiles[FContext].setFunctionHash(*Checksum);
}
return sampleprof_error::success;
}
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index 71dba6281f76..d3bc05e06fdf 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -147,7 +147,7 @@ std::error_code SampleProfileWriterExtBinaryBase::write(
std::error_code
SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
uint64_t Offset = OutputStream->tell();
- StringRef Name = S.getName();
+ StringRef Name = S.getNameWithContext(true);
FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
encodeULEB128(S.getHeadSamples(), *OutputStream);
return writeBody(S);
@@ -635,7 +635,7 @@ std::error_code SampleProfileWriterBinary::writeSummary() {
std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
auto &OS = *OutputStream;
- if (std::error_code EC = writeNameIdx(S.getName()))
+ if (std::error_code EC = writeNameIdx(S.getNameWithContext(true)))
return EC;
encodeULEB128(S.getTotalSamples(), OS);
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
index 37fc27e91100..660d79de667c 100644
--- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -179,7 +179,7 @@ SampleContextTracker::SampleContextTracker(
SampleContext Context(FuncSample.first(), RawContext);
LLVM_DEBUG(dbgs() << "Tracking Context for function: " << Context << "\n");
if (!Context.isBaseContext())
- FuncToCtxtProfileSet[Context.getName()].insert(FSamples);
+ FuncToCtxtProfileSet[Context.getNameWithoutContext()].insert(FSamples);
ContextTrieNode *NewNode = getOrCreateContextPath(Context, true);
assert(!NewNode->getFunctionSamples() &&
"New node can't have sample profile");
diff --git a/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll b/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
index ed32c2a0027b..adda7022047d 100644
--- a/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
@@ -1,18 +1,22 @@
; Test for CSSPGO's SampleContextTracker to make sure context profile tree is promoted and merged properly
; based on inline decision, so post inline counts are accurate.
+; RUN: llvm-profdata merge --sample --extbinary %S/Inputs/profile-context-tracker.prof -o %t
+
; Note that we need new pass manager to enable top-down processing for sample profile loader
; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile
; main:3 @ _Z5funcAi
; main:3 @ _Z5funcAi:1 @ _Z8funcLeafi
; _Z5funcBi:1 @ _Z8funcLeafi
; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL
; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile
; main:3 @ _Z5funcAi
; _Z5funcAi:1 @ _Z8funcLeafi
; _Z5funcBi:1 @ _Z8funcLeafi
; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT
@factor = dso_local global i32 3, align 4, !dbg !0
diff --git a/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext b/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext
new file mode 100644
index 000000000000..eead4d4d62f0
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext
@@ -0,0 +1,36 @@
+[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]:1467299:11
+ 0: 6
+ 1: 6
+ 3: 287884
+ 4: 287864 _Z3fibi:315608
+ 15: 23
+[main:3.1 @ _Z5funcBi:1 @ _Z8funcLeafi]:500853:20
+ 0: 15
+ 1: 15
+ 3: 74946
+ 4: 74941 _Z3fibi:82359
+ 10: 23324
+ 11: 23327 _Z3fibi:25228
+ 15: 11
+[main]:154:0
+ 2: 12
+ 3: 18 _Z5funcAi:11
+ 3.1: 18 _Z5funcBi:19
+[external:12 @ main]:154:12
+ 2: 12
+ 3: 10 _Z5funcAi:7
+ 3.1: 10 _Z5funcBi:11
+[main:3.1 @ _Z5funcBi]:120:19
+ 0: 19
+ 1: 19 _Z8funcLeafi:20
+ 3: 12
+[externalA:17 @ _Z5funcBi]:120:3
+ 0: 3
+ 1: 3
+[external:10 @ _Z5funcBi]:120:10
+ 0: 10
+ 1: 10
+[main:3 @ _Z5funcAi]:99:11
+ 0: 10
+ 1: 10 _Z8funcLeafi:11
+ 3: 24
diff --git a/llvm/test/tools/llvm-profdata/cs-sample-profile.test b/llvm/test/tools/llvm-profdata/cs-sample-profile.test
new file mode 100644
index 000000000000..04c573ddece3
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/cs-sample-profile.test
@@ -0,0 +1,4 @@
+RUN: llvm-profdata merge --sample --text -output=%t.proftext %S/Inputs/cs-sample.proftext
+RUN:
diff -b %t.proftext %S/Inputs/cs-sample.proftext
+RUN: llvm-profdata merge --sample --extbinary %p/Inputs/cs-sample.proftext -o %t.prof && llvm-profdata merge --sample --text %t.prof -o %t1.proftext
+RUN:
diff -b %t1.proftext %S/Inputs/cs-sample.proftext
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 8dc43924c067..7e53c30c7579 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -696,7 +696,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
Remapper ? remapSamples(I->second, *Remapper, Result)
: FunctionSamples();
FunctionSamples &Samples = Remapper ? Remapped : I->second;
- StringRef FName = Samples.getName();
+ StringRef FName = Samples.getNameWithContext(true);
MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight));
if (Result != sampleprof_error::success) {
std::error_code EC = make_error_code(Result);
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index 265beccb84a8..7624fd3f2808 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -164,7 +164,7 @@ CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr) {
if (Ret.second) {
SampleContext FContext(Ret.first->first(), RawContext);
FunctionSamples &FProfile = Ret.first->second;
- FProfile.setName(FContext.getName());
+ FProfile.setName(FContext.getNameWithoutContext());
FProfile.setContext(FContext);
}
return Ret.first->second;
More information about the llvm-commits
mailing list