[llvm] 00ef28e - [CSSPGO] Fix dangling context strings and improve profile order consistency and error handling
Wenlei He via llvm-commits
llvm-commits at lists.llvm.org
Sat Apr 10 12:58:55 PDT 2021
Author: Wenlei He
Date: 2021-04-10T12:39:10-07:00
New Revision: 00ef28ef21f0a46059203c5aa96e9febe1192e43
URL: https://github.com/llvm/llvm-project/commit/00ef28ef21f0a46059203c5aa96e9febe1192e43
DIFF: https://github.com/llvm/llvm-project/commit/00ef28ef21f0a46059203c5aa96e9febe1192e43.diff
LOG: [CSSPGO] Fix dangling context strings and improve profile order consistency and error handling
This patch fixed the following issues along side with some refactoring:
1. Fix bugs where StringRef for context string out live the underlying std::string. We now keep string table in profile generator to hold std::strings. We also do the same for bracketed context strings in profile writer.
2. Make sure profile output strictly follow (total sample, name) order. Previously, there's inconsistency between ProfileMap's key and FunctionSamples's name, leading to inconsistent ordering. This is now fixed by introducing context profile canonicalization. Assertions are also added to make sure ProfileMap's key and FunctionSamples's name are always consistent.
3. Enhanced error handling for profile writing to make sure we bubble up errors properly for both llvm-profgen and llvm-profdata when string table is not populated correctly for extended binary profile.
4. Keep all internal context representation bracket free. This avoids creating new strings for context trimming, merging and preinline. getNameWithContext API is now simplied accordingly.
5. Factor out the code for context trimming and merging into SampleContextTrimmer in SampleProf.cpp. This enables llvm-profdata to use the trimmer when merging profiles. Changes in llvm-profgen will be in separate patch.
Differential Revision: https://reviews.llvm.org/D100090
Added:
Modified:
llvm/include/llvm/ProfileData/SampleProf.h
llvm/include/llvm/ProfileData/SampleProfWriter.h
llvm/lib/ProfileData/SampleProf.cpp
llvm/lib/ProfileData/SampleProfWriter.cpp
llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test
llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test
llvm/tools/llvm-profdata/llvm-profdata.cpp
llvm/tools/llvm-profgen/CSPreInliner.cpp
llvm/tools/llvm-profgen/ProfileGenerator.cpp
llvm/tools/llvm-profgen/ProfileGenerator.h
Removed:
################################################################################
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 8b590e84dd9be..4fda543a03ec4 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -462,9 +462,7 @@ class SampleContext {
bool isBaseContext() const { return CallingContext.empty(); }
StringRef getNameWithoutContext() const { return Name; }
StringRef getCallingContext() const { return CallingContext; }
- StringRef getNameWithContext(bool WithBracket = false) const {
- return WithBracket ? InputContext : FullContext;
- }
+ StringRef getNameWithContext() const { return FullContext; }
private:
// Give a context string, decode and populate internal states like
@@ -472,7 +470,6 @@ class SampleContext {
// `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
void setContext(StringRef ContextStr, ContextStateMask CState) {
assert(!ContextStr.empty());
- InputContext = ContextStr;
// Note that `[]` wrapped input indicates a full context string, otherwise
// it's treated as context-less function name only.
bool HasContext = ContextStr.startswith("[");
@@ -504,9 +501,6 @@ class SampleContext {
}
}
- // Input context string including bracketed calling context and leaf function
- // name
- StringRef InputContext;
// Full context string including calling context and leaf function name
StringRef FullContext;
// Function name for the associated sample profile
@@ -711,7 +705,7 @@ class FunctionSamples {
Name = Other.getName();
if (!GUIDToFuncNameMap)
GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
- if (Context.getNameWithContext(true).empty())
+ if (Context.getNameWithContext().empty())
Context = Other.getContext();
if (FunctionHash == 0) {
// Set the function hash code for the target profile.
@@ -780,10 +774,8 @@ class FunctionSamples {
StringRef getName() const { return Name; }
/// Return function name with context.
- StringRef getNameWithContext(bool WithBracket = false) const {
- return FunctionSamples::ProfileIsCS
- ? Context.getNameWithContext(WithBracket)
- : Name;
+ StringRef getNameWithContext() const {
+ return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name;
}
/// Return the original function name.
@@ -986,6 +978,24 @@ template <class LocationT, class SampleT> class SampleSorter {
SamplesWithLocList V;
};
+/// SampleContextTrimmer impelements helper functions to trim, merge cold
+/// context profiles. It also supports context profile canonicalization to make
+/// sure ProfileMap's key is consistent with FunctionSample's name/context.
+class SampleContextTrimmer {
+public:
+ SampleContextTrimmer(StringMap<FunctionSamples> &Profiles)
+ : ProfileMap(Profiles){};
+ // Trim and merge cold context profile when requested.
+ void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold,
+ bool TrimColdContext = true,
+ bool MergeColdContext = true);
+ // Canonicalize context profile name and attributes.
+ void canonicalizeContextProfiles();
+
+private:
+ StringMap<FunctionSamples> &ProfileMap;
+};
+
/// ProfileSymbolList records the list of function symbols shown up
/// in the binary used to generate the profile. It is useful to
/// to discriminate a function being so cold as not to shown up
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 419ebd6eb7ae2..107f7a730a3cc 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -25,6 +25,7 @@
#include <memory>
#include <set>
#include <system_error>
+#include <unordered_set>
namespace llvm {
namespace sampleprof {
@@ -136,13 +137,14 @@ class SampleProfileWriterBinary : public SampleProfileWriter {
virtual std::error_code
writeHeader(const StringMap<FunctionSamples> &ProfileMap) override;
std::error_code writeSummary();
- std::error_code writeNameIdx(StringRef FName);
+ std::error_code writeNameIdx(StringRef FName, bool IsContextName = false);
std::error_code writeBody(const FunctionSamples &S);
inline void stablizeNameTable(std::set<StringRef> &V);
MapVector<StringRef, uint32_t> NameTable;
+ std::unordered_set<std::string> BracketedContextStr;
- void addName(StringRef FName);
+ void addName(StringRef FName, bool IsContextName = false);
void addNames(const FunctionSamples &S);
private:
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 6647b26373411..565d67eed0450 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -316,6 +316,85 @@ std::error_code ProfileSymbolList::read(const uint8_t *Data,
return sampleprof_error::success;
}
+void SampleContextTrimmer::trimAndMergeColdContextProfiles(
+ uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext) {
+ if (!TrimColdContext && !MergeColdContext)
+ return;
+
+ // Nothing to merge if sample threshold is zero
+ if (ColdCountThreshold == 0)
+ return;
+
+ // Filter the cold profiles from ProfileMap and move them into a tmp
+ // container
+ std::vector<std::pair<StringRef, const FunctionSamples *>> ColdProfiles;
+ for (const auto &I : ProfileMap) {
+ const FunctionSamples &FunctionProfile = I.second;
+ if (FunctionProfile.getTotalSamples() >= ColdCountThreshold)
+ continue;
+ ColdProfiles.emplace_back(I.getKey(), &I.second);
+ }
+
+ // Remove the cold profile from ProfileMap and merge them into BaseProileMap
+ StringMap<FunctionSamples> BaseProfileMap;
+ for (const auto &I : ColdProfiles) {
+ if (MergeColdContext) {
+ auto Ret = BaseProfileMap.try_emplace(
+ I.second->getContext().getNameWithoutContext(), FunctionSamples());
+ FunctionSamples &BaseProfile = Ret.first->second;
+ BaseProfile.merge(*I.second);
+ }
+ ProfileMap.erase(I.first);
+ }
+
+ // Merge the base profiles into ProfileMap;
+ for (const auto &I : BaseProfileMap) {
+ // Filter the cold base profile
+ if (TrimColdContext && I.second.getTotalSamples() < ColdCountThreshold &&
+ ProfileMap.find(I.getKey()) == ProfileMap.end())
+ continue;
+ // Merge the profile if the original profile exists, otherwise just insert
+ // as a new profile
+ auto Ret = ProfileMap.try_emplace(I.getKey(), FunctionSamples());
+ if (Ret.second) {
+ SampleContext FContext(Ret.first->first(), RawContext);
+ FunctionSamples &FProfile = Ret.first->second;
+ FProfile.setContext(FContext);
+ FProfile.setName(FContext.getNameWithoutContext());
+ }
+ FunctionSamples &OrigProfile = Ret.first->second;
+ OrigProfile.merge(I.second);
+ }
+}
+
+void SampleContextTrimmer::canonicalizeContextProfiles() {
+ StringSet<> ProfilesToBeRemoved;
+ // Note that StringMap order is guaranteed to be top-down order,
+ // this makes sure we make room for promoted/merged context in the
+ // map, before we move profiles in the map.
+ for (auto &I : ProfileMap) {
+ FunctionSamples &FProfile = I.second;
+ StringRef ContextStr = FProfile.getNameWithContext();
+ if (I.first() == ContextStr)
+ continue;
+
+ // Use the context string from FunctionSamples to update the keys of
+ // ProfileMap. They can get out of sync after context profile promotion
+ // through pre-inliner.
+ auto Ret = ProfileMap.try_emplace(ContextStr, FProfile);
+ assert(Ret.second && "Conext conflict during canonicalization");
+ FProfile = Ret.first->second;
+
+ // Track the context profile to remove
+ ProfilesToBeRemoved.erase(ContextStr);
+ ProfilesToBeRemoved.insert(I.first());
+ }
+
+ for (auto &I : ProfilesToBeRemoved) {
+ ProfileMap.erase(I.first());
+ }
+}
+
std::error_code ProfileSymbolList::write(raw_ostream &OS) {
// Sort the symbols before output. If doing compression.
// It will make the compression much more effective.
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index d32fcbf7912ef..43cfb9380a9e3 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -46,9 +46,11 @@ std::error_code SampleProfileWriter::writeFuncProfiles(
// Sort the ProfileMap by total samples.
typedef std::pair<StringRef, const FunctionSamples *> NameFunctionSamples;
std::vector<NameFunctionSamples> V;
- for (const auto &I : ProfileMap)
- V.push_back(std::make_pair(I.getKey(), &I.second));
-
+ for (const auto &I : ProfileMap) {
+ assert(I.getKey() == I.second.getNameWithContext() &&
+ "Inconsistent profile map");
+ V.push_back(std::make_pair(I.second.getNameWithContext(), &I.second));
+ }
llvm::stable_sort(
V, [](const NameFunctionSamples &A, const NameFunctionSamples &B) {
if (A.second->getTotalSamples() == B.second->getTotalSamples())
@@ -147,7 +149,7 @@ std::error_code SampleProfileWriterExtBinaryBase::write(
std::error_code
SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
uint64_t Offset = OutputStream->tell();
- StringRef Name = S.getNameWithContext(true);
+ StringRef Name = S.getNameWithContext();
FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
encodeULEB128(S.getHeadSamples(), *OutputStream);
return writeBody(S);
@@ -160,9 +162,11 @@ std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
encodeULEB128(FuncOffsetTable.size(), OS);
// Write out FuncOffsetTable.
- for (auto entry : FuncOffsetTable) {
- writeNameIdx(entry.first);
- encodeULEB128(entry.second, OS);
+ for (auto Entry : FuncOffsetTable) {
+ if (std::error_code EC =
+ writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS))
+ return EC;
+ encodeULEB128(Entry.second, OS);
}
FuncOffsetTable.clear();
return sampleprof_error::success;
@@ -174,7 +178,9 @@ std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
return sampleprof_error::success;
auto &OS = *OutputStream;
for (const auto &Entry : Profiles) {
- writeNameIdx(Entry.first());
+ if (std::error_code EC = writeNameIdx(Entry.second.getNameWithContext(),
+ FunctionSamples::ProfileIsCS))
+ return EC;
if (FunctionSamples::ProfileIsProbeBased)
encodeULEB128(Entry.second.getFunctionHash(), OS);
if (FunctionSamples::ProfileIsCS)
@@ -204,7 +210,9 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
const StringMap<FunctionSamples> &ProfileMap) {
for (const auto &I : ProfileMap) {
- addName(I.first());
+ assert(I.first() == I.second.getNameWithContext() &&
+ "Inconsistent profile map");
+ addName(I.second.getNameWithContext(), FunctionSamples::ProfileIsCS);
addNames(I.second);
}
@@ -378,7 +386,11 @@ std::error_code SampleProfileWriterCompactBinary::write(
/// it needs to be parsed by the SampleProfileReaderText class.
std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
auto &OS = *OutputStream;
- OS << S.getNameWithContext(true) << ":" << S.getTotalSamples();
+ if (FunctionSamples::ProfileIsCS)
+ OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples();
+ else
+ OS << S.getName() << ":" << S.getTotalSamples();
+
if (Indent == 0)
OS << ":" << S.getHeadSamples();
OS << "\n";
@@ -431,15 +443,26 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
return sampleprof_error::success;
}
-std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
- const auto &ret = NameTable.find(FName);
- if (ret == NameTable.end())
+std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName,
+ bool IsContextName) {
+ std::string BracketedName;
+ if (IsContextName) {
+ BracketedName = "[" + FName.str() + "]";
+ FName = StringRef(BracketedName);
+ }
+
+ const auto &Ret = NameTable.find(FName);
+ if (Ret == NameTable.end())
return sampleprof_error::truncated_name_table;
- encodeULEB128(ret->second, *OutputStream);
+ encodeULEB128(Ret->second, *OutputStream);
return sampleprof_error::success;
}
-void SampleProfileWriterBinary::addName(StringRef FName) {
+void SampleProfileWriterBinary::addName(StringRef FName, bool IsContextName) {
+ if (IsContextName) {
+ auto It = BracketedContextStr.insert("[" + FName.str() + "]");
+ FName = StringRef(*It.first);
+ }
NameTable.insert(std::make_pair(FName, 0));
}
@@ -500,9 +523,11 @@ std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() {
encodeULEB128(FuncOffsetTable.size(), OS);
// Write out FuncOffsetTable.
- for (auto entry : FuncOffsetTable) {
- writeNameIdx(entry.first);
- encodeULEB128(entry.second, OS);
+ for (auto Entry : FuncOffsetTable) {
+ if (std::error_code EC =
+ writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS))
+ return EC;
+ encodeULEB128(Entry.second, OS);
}
return sampleprof_error::success;
}
@@ -539,7 +564,9 @@ std::error_code SampleProfileWriterBinary::writeHeader(
// Generate the name table for all the functions referenced in the profile.
for (const auto &I : ProfileMap) {
- addName(I.first());
+ assert(I.first() == I.second.getNameWithContext() &&
+ "Inconsistent profile map");
+ addName(I.first(), FunctionSamples::ProfileIsCS);
addNames(I.second);
}
@@ -654,7 +681,8 @@ std::error_code SampleProfileWriterBinary::writeSummary() {
std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
auto &OS = *OutputStream;
- if (std::error_code EC = writeNameIdx(S.getNameWithContext(true)))
+ if (std::error_code EC =
+ writeNameIdx(S.getNameWithContext(), FunctionSamples::ProfileIsCS))
return EC;
encodeULEB128(S.getTotalSamples(), OS);
diff --git a/llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test b/llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test
index 15bdd870879e8..f002a09532e34 100644
--- a/llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test
+++ b/llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test
@@ -10,16 +10,16 @@
; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa]:24:0
; CHECK-UNCOMPRESS: 1: 1
; CHECK-UNCOMPRESS: 2: 13 fb:11
-; CHECK-UNCOMPRESS:[main:1 @ foo]:7:0
-; CHECK-UNCOMPRESS: 2: 1
-; CHECK-UNCOMPRESS: 3: 2 fa:1
; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa:2 @ fb:2 @ fa]:7:0
; CHECK-UNCOMPRESS: 1: 1
; CHECK-UNCOMPRESS: 2: 2 fb:1
-; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa:2 @ fb:2 @ fa:2 @ fb]:2:0
-; CHECK-UNCOMPRESS: 2: 1 fa:1
+; CHECK-UNCOMPRESS:[main:1 @ foo]:7:0
+; CHECK-UNCOMPRESS: 2: 1
+; CHECK-UNCOMPRESS: 3: 2 fa:1
; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa:2 @ fb:2 @ fa:2 @ fb:2 @ fa]:2:0
; CHECK-UNCOMPRESS: 4: 1
+; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa:2 @ fb:2 @ fa:2 @ fb]:2:0
+; CHECK-UNCOMPRESS: 2: 1 fa:1
; CHECK: [main:1 @ foo:3 @ fa:2 @ fb]:48:0
diff --git a/llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test b/llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test
index 0936e5d615caf..3345721722f8f 100644
--- a/llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test
+++ b/llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test
@@ -4,11 +4,11 @@
; RUN: llvm-profgen --perfscript=%S/Inputs/recursion-compression-pseudoprobe.perfscript --binary=%S/Inputs/recursion-compression-pseudoprobe.perfbin --output=%t --show-unwinder-output --csprof-cold-thres=0 | FileCheck %s --check-prefix=CHECK-UNWINDER
; RUN: FileCheck %s --input-file %t
-; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa]:4:1
+; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa]:4:1
; CHECK-UNCOMPRESS: 1: 1
; CHECK-UNCOMPRESS: 3: 1
-; CHECK-UNCOMPRESS: 5: 1
-; CHECK-UNCOMPRESS: 8: 1 fa:1
+; CHECK-UNCOMPRESS: 4: 1
+; CHECK-UNCOMPRESS: 7: 1 fb:1
; CHECK-UNCOMPRESS: !CFGChecksum: 120515930909
; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa]:4:1
; CHECK-UNCOMPRESS: 1: 1
@@ -16,28 +16,13 @@
; CHECK-UNCOMPRESS: 4: 1
; CHECK-UNCOMPRESS: 7: 1 fb:1
; CHECK-UNCOMPRESS: !CFGChecksum: 120515930909
-; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa]:4:1
+; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa]:4:1
; CHECK-UNCOMPRESS: 1: 1
; CHECK-UNCOMPRESS: 3: 1
-; CHECK-UNCOMPRESS: 4: 1
-; CHECK-UNCOMPRESS: 7: 1 fb:1
+; CHECK-UNCOMPRESS: 5: 1
+; CHECK-UNCOMPRESS: 8: 1 fa:1
; CHECK-UNCOMPRESS: !CFGChecksum: 120515930909
-; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1
-; CHECK-UNCOMPRESS: 1: 1
-; CHECK-UNCOMPRESS: 2: 1
-; CHECK-UNCOMPRESS: 5: 1 fb:1
-; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756
-; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1
-; CHECK-UNCOMPRESS: 1: 1
-; CHECK-UNCOMPRESS: 2: 1
-; CHECK-UNCOMPRESS: 5: 1 fb:1
-; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756
-; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1
-; CHECK-UNCOMPRESS: 1: 1
-; CHECK-UNCOMPRESS: 2: 1
-; CHECK-UNCOMPRESS: 5: 1 fb:1
-; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756
-; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1
+; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb]:3:1
; CHECK-UNCOMPRESS: 1: 1
; CHECK-UNCOMPRESS: 3: 1
; CHECK-UNCOMPRESS: 6: 1 fa:1
@@ -47,11 +32,26 @@
; CHECK-UNCOMPRESS: 3: 1
; CHECK-UNCOMPRESS: 6: 1 fa:1
; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756
-; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb]:3:1
+; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1
; CHECK-UNCOMPRESS: 1: 1
; CHECK-UNCOMPRESS: 3: 1
; CHECK-UNCOMPRESS: 6: 1 fa:1
; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756
+; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1
+; CHECK-UNCOMPRESS: 1: 1
+; CHECK-UNCOMPRESS: 2: 1
+; CHECK-UNCOMPRESS: 5: 1 fb:1
+; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756
+; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1
+; CHECK-UNCOMPRESS: 1: 1
+; CHECK-UNCOMPRESS: 2: 1
+; CHECK-UNCOMPRESS: 5: 1 fb:1
+; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756
+; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1
+; CHECK-UNCOMPRESS: 1: 1
+; CHECK-UNCOMPRESS: 2: 1
+; CHECK-UNCOMPRESS: 5: 1 fb:1
+; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756
; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb:6 @ fa]:2:1
; CHECK-UNCOMPRESS: 1: 1
; CHECK-UNCOMPRESS: 3: 1
@@ -74,24 +74,24 @@
; CHECK: 4: 1
; CHECK: 7: 1 fb:1
; CHECK: !CFGChecksum: 120515930909
-; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa]:4:1
-; CHECK: 1: 1
-; CHECK: 3: 1
-; CHECK: 5: 1
-; CHECK: 8: 1 fa:1
-; CHECK: !CFGChecksum: 120515930909
; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa]:4:1
; CHECK: 1: 1
; CHECK: 3: 1
; CHECK: 4: 1
; CHECK: 7: 1 fb:1
; CHECK: !CFGChecksum: 120515930909
-; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb]:3:1
+; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa]:4:1
+; CHECK: 1: 1
+; CHECK: 3: 1
+; CHECK: 5: 1
+; CHECK: 8: 1 fa:1
+; CHECK: !CFGChecksum: 120515930909
+; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb]:3:1
; CHECK: 1: 1
; CHECK: 3: 1
; CHECK: 6: 1 fa:1
; CHECK: !CFGChecksum: 72617220756
-; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb]:3:1
+; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb]:3:1
; CHECK: 1: 1
; CHECK: 3: 1
; CHECK: 6: 1 fa:1
@@ -99,6 +99,7 @@
+
; CHECK-UNWINDER: Binary(recursion-compression-pseudoprobe.perfbin)'s Range Counter:
; CHECK-UNWINDER: main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5
; CHECK-UNWINDER: (7a0, 7a7): 1
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index f274f5a911066..66b95d1ba1d90 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -710,7 +710,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
Remapper ? remapSamples(I->second, *Remapper, Result)
: FunctionSamples();
FunctionSamples &Samples = Remapper ? Remapped : I->second;
- StringRef FName = Samples.getNameWithContext(true);
+ StringRef FName = Samples.getNameWithContext();
MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight));
if (Result != sampleprof_error::success) {
std::error_code EC = make_error_code(Result);
@@ -734,7 +734,8 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
auto Buffer = getInputFileBuf(ProfileSymbolListFile);
handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList,
CompressAllSections, UseMD5, GenPartialProfile);
- Writer->write(ProfileMap);
+ if (std::error_code EC = Writer->write(ProfileMap))
+ exitWithErrorCode(std::move(EC));
}
static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) {
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp
index 842794d1016a9..50d8dd34ffa84 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.cpp
+++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp
@@ -225,5 +225,8 @@ void CSPreInliner::run() {
ProfileMap.erase(ContextName);
}
+ // Make sure ProfileMap's key is consistent with FunctionSamples' name.
+ SampleContextTrimmer(ProfileMap).canonicalizeContextProfiles();
+
LLVM_DEBUG(printProfileNames(ProfileMap, false));
}
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index a6794f01551e8..9e1ba9bca1b8b 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -86,7 +86,8 @@ ProfileGenerator::create(const BinarySampleCounterMap &BinarySampleCounters,
void ProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer,
StringMap<FunctionSamples> &ProfileMap) {
- Writer->write(ProfileMap);
+ if (std::error_code EC = Writer->write(ProfileMap))
+ exitWithError(std::move(EC));
}
void ProfileGenerator::write() {
@@ -198,7 +199,10 @@ CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr,
bool WasLeafInlined) {
auto Ret = ProfileMap.try_emplace(ContextStr, FunctionSamples());
if (Ret.second) {
- SampleContext FContext(Ret.first->first(), RawContext);
+ // Make a copy of the underlying context string in string table
+ // before StringRef wrapper is used for context.
+ auto It = ContextStrings.insert(ContextStr.str());
+ SampleContext FContext(*It.first, RawContext);
if (WasLeafInlined)
FContext.setAttribute(ContextWasInlined);
FunctionSamples &FProfile = Ret.first->second;
@@ -401,84 +405,27 @@ void CSProfileGenerator::postProcessProfiles() {
PSI->getColdCountThreshold())
.run();
- mergeAndTrimColdProfile(ProfileMap);
+ // Trim and merge cold context profile using cold threshold above;
+ SampleContextTrimmer(ProfileMap)
+ .trimAndMergeColdContextProfiles(
+ CSProfColdThreshold, CSProfTrimColdContext, CSProfMergeColdContext);
}
void CSProfileGenerator::computeSummaryAndThreshold() {
SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
auto Summary = Builder.computeSummaryForProfiles(ProfileMap);
PSI.reset(new ProfileSummaryInfo(std::move(Summary)));
-}
-
-void CSProfileGenerator::mergeAndTrimColdProfile(
- StringMap<FunctionSamples> &ProfileMap) {
- if (!CSProfMergeColdContext && !CSProfTrimColdContext)
- return;
-
- // Use threshold calculated from profile summary unless specified
- uint64_t ColdThreshold = PSI->getColdCountThreshold();
- if (CSProfColdThreshold.getNumOccurrences()) {
- ColdThreshold = CSProfColdThreshold;
- }
- // Nothing to merge if sample threshold is zero
- if (ColdThreshold == 0)
- return;
-
- // Filter the cold profiles from ProfileMap and move them into a tmp
- // container
- std::vector<std::pair<StringRef, const FunctionSamples *>> ColdProfiles;
- for (const auto &I : ProfileMap) {
- const FunctionSamples &FunctionProfile = I.second;
- if (FunctionProfile.getTotalSamples() >= ColdThreshold)
- continue;
- ColdProfiles.emplace_back(I.getKey(), &I.second);
- }
-
- // Remove the code profile from ProfileMap and merge them into BaseProileMap
- StringMap<FunctionSamples> BaseProfileMap;
- for (const auto &I : ColdProfiles) {
- if (CSProfMergeColdContext) {
- auto Ret = BaseProfileMap.try_emplace(
- I.second->getContext().getNameWithoutContext(), FunctionSamples());
- FunctionSamples &BaseProfile = Ret.first->second;
- BaseProfile.merge(*I.second);
- }
- ProfileMap.erase(I.first);
- }
-
- // Merge the base profiles into ProfileMap;
- for (const auto &I : BaseProfileMap) {
- // Filter the cold base profile
- if (CSProfTrimColdContext &&
- I.second.getTotalSamples() < CSProfColdThreshold &&
- ProfileMap.find(I.getKey()) == ProfileMap.end())
- continue;
- // Merge the profile if the original profile exists, otherwise just insert
- // as a new profile
- FunctionSamples &OrigProfile = getFunctionProfileForContext(I.getKey());
- OrigProfile.merge(I.second);
+ // Use threshold calculated from profile summary unless specified.
+ if (!CSProfColdThreshold.getNumOccurrences()) {
+ CSProfColdThreshold = PSI->getColdCountThreshold();
}
}
void CSProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer,
StringMap<FunctionSamples> &ProfileMap) {
- // Add bracket for context key to support
diff erent profile binary format
- StringMap<FunctionSamples> CxtWithBracketPMap;
- for (const auto &Item : ProfileMap) {
- // After CSPreInliner the key of ProfileMap is no longer accurate for
- // context, use the context attached to function samples instead.
- std::string ContextWithBracket =
- "[" + Item.second.getNameWithContext().str() + "]";
- auto Ret = CxtWithBracketPMap.try_emplace(ContextWithBracket, Item.second);
- assert(Ret.second && "Must be a unique context");
- SampleContext FContext(Ret.first->first(), RawContext);
- FunctionSamples &FProfile = Ret.first->second;
- FContext.setAllAttributes(FProfile.getContext().getAllAttributes());
- FProfile.setName(FContext.getNameWithoutContext());
- FProfile.setContext(FContext);
- }
- Writer->write(CxtWithBracketPMap);
+ if (std::error_code EC = Writer->write(ProfileMap))
+ exitWithError(std::move(EC));
}
// Helper function to extract context prefix string stack
@@ -590,7 +537,7 @@ void PseudoProbeCSProfileGenerator::populateBodySamplesWithProbes(
// context id to infer caller's context id to ensure they share the
// same context prefix.
StringRef CalleeContextId =
- FunctionProfile.getContext().getNameWithContext(true);
+ FunctionProfile.getContext().getNameWithContext();
StringRef CallerContextId;
FrameLocation &&CallerLeafFrameLoc =
getCallerContext(CalleeContextId, CallerContextId);
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
index 0ba884f3afbb8..b2e2a339a26b0 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.h
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -15,6 +15,7 @@
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/ProfileData/SampleProfWriter.h"
#include <memory>
+#include <unordered_set>
using namespace llvm;
using namespace sampleprof;
@@ -182,9 +183,6 @@ class CSProfileGenerator : public ProfileGenerator {
// Post processing for profiles before writing out, such as mermining
// and trimming cold profiles, running preinliner on profiles.
void postProcessProfiles();
- // Merge cold context profile whose total sample is below threshold
- // into base profile.
- void mergeAndTrimColdProfile(StringMap<FunctionSamples> &ProfileMap);
void computeSummaryAndThreshold();
void write(std::unique_ptr<SampleProfileWriter> Writer,
StringMap<FunctionSamples> &ProfileMap) override;
@@ -192,6 +190,9 @@ class CSProfileGenerator : public ProfileGenerator {
// Profile summary to answer isHotCount and isColdCount queries.
std::unique_ptr<ProfileSummaryInfo> PSI;
+ // String table owning context strings created from profile generation.
+ std::unordered_set<std::string> ContextStrings;
+
private:
// Helper function for updating body sample for a leaf location in
// FunctionProfile
More information about the llvm-commits
mailing list