[llvm] b49eac7 - Recommit [SampleFDO] Add flag for partial profile.
Wei Mi via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 7 14:30:33 PDT 2020
Author: Wei Mi
Date: 2020-04-07T14:28:25-07:00
New Revision: b49eac71ad76cd8ee3771745ff591ee8771eab1b
URL: https://github.com/llvm/llvm-project/commit/b49eac71ad76cd8ee3771745ff591ee8771eab1b
DIFF: https://github.com/llvm/llvm-project/commit/b49eac71ad76cd8ee3771745ff591ee8771eab1b.diff
LOG: Recommit [SampleFDO] Add flag for partial profile.
Fix the error of show-prof-info.test on some platforms without zlib.
The common profile usage is to collect profile from a target and then use the profile to guide the optimized build for the same target. There are some cases that no profile can be collected for a target. In those cases, although no full profile is available, it is possible to have some partial profile collected from other targets to optimize common libraries and utilities. A flag is needed to tell the partial profile from the full profile apart, so compiler can use different strategy for them.
Differential Revision: https://reviews.llvm.org/D77426
Added:
llvm/test/tools/llvm-profdata/show-prof-info.test
Modified:
llvm/include/llvm/IR/ProfileSummary.h
llvm/include/llvm/ProfileData/SampleProf.h
llvm/include/llvm/ProfileData/SampleProfWriter.h
llvm/lib/ProfileData/SampleProfReader.cpp
llvm/tools/llvm-profdata/llvm-profdata.cpp
Removed:
llvm/test/tools/llvm-profdata/show-prof-size.test
################################################################################
diff --git a/llvm/include/llvm/IR/ProfileSummary.h b/llvm/include/llvm/IR/ProfileSummary.h
index 78635ec4386c..7532b44edd98 100644
--- a/llvm/include/llvm/IR/ProfileSummary.h
+++ b/llvm/include/llvm/IR/ProfileSummary.h
@@ -49,6 +49,12 @@ class ProfileSummary {
SummaryEntryVector DetailedSummary;
uint64_t TotalCount, MaxCount, MaxInternalCount, MaxFunctionCount;
uint32_t NumCounts, NumFunctions;
+ /// If 'Partial' is false, it means the profile being used to optimize
+ /// a target is collected from the same target.
+ /// If 'Partial' is true, it means the profile is for common/shared
+ /// code. The common profile is usually merged from profiles collected
+ /// from running other targets.
+ bool Partial = false;
/// Return detailed summary as metadata.
Metadata *getDetailedSummaryMD(LLVMContext &Context);
@@ -76,6 +82,8 @@ class ProfileSummary {
uint64_t getTotalCount() { return TotalCount; }
uint64_t getMaxCount() { return MaxCount; }
uint64_t getMaxInternalCount() { return MaxInternalCount; }
+ void setPartialProfile(bool PP) { Partial = PP; }
+ bool isPartialProfile() { return Partial; }
};
} // end namespace llvm
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 7b1086c74919..f1ddbe3ba579 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -169,6 +169,13 @@ enum class SecNameTableFlags : uint32_t {
SecFlagInValid = 0,
SecFlagMD5Name = (1 << 0)
};
+enum class SecProfSummaryFlags : uint32_t {
+ SecFlagInValid = 0,
+ /// SecFlagPartial means the profile is for common/shared code.
+ /// The common profile is usually merged from profiles collected
+ /// from running other targets.
+ SecFlagPartial = (1 << 0)
+};
// Verify section specific flag is used for the correct section.
template <class SecFlagType>
@@ -183,6 +190,9 @@ static inline void verifySecFlag(SecType Type, SecFlagType Flag) {
case SecNameTable:
IsFlagLegal = std::is_same<SecNameTableFlags, SecFlagType>();
break;
+ case SecProfSummary:
+ IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>();
+ break;
default:
break;
}
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 54500f513ad1..7d0df9e44f58 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -59,6 +59,7 @@ class SampleProfileWriter {
virtual void setProfileSymbolList(ProfileSymbolList *PSL) {}
virtual void setToCompressAllSections() {}
virtual void setUseMD5() {}
+ virtual void setPartialProfile() {}
protected:
SampleProfileWriter(std::unique_ptr<raw_ostream> &OS)
@@ -217,6 +218,13 @@ class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name);
}
+ // Set the profile to be partial. It means the profile is for
+ // common/shared code. The common profile is usually merged from
+ // profiles collected from running other targets.
+ virtual void setPartialProfile() override {
+ addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagPartial);
+ }
+
private:
virtual void initSectionHdrLayout() override {
// Note that SecFuncOffsetTable section is written after SecLBRProfile
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 33efb51dca07..e27bf0771b14 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -478,6 +478,8 @@ std::error_code SampleProfileReaderExtBinary::readOneSection(
case SecProfSummary:
if (std::error_code EC = readSummary())
return EC;
+ if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))
+ Summary->setPartialProfile(true);
break;
case SecNameTable:
if (std::error_code EC = readNameTableSec(
@@ -831,11 +833,40 @@ uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
return FileSize;
}
+static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
+ std::string Flags;
+ if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))
+ Flags.append("{compressed,");
+ else
+ Flags.append("{");
+
+ switch (Entry.Type) {
+ case SecNameTable:
+ if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name))
+ Flags.append("md5,");
+ break;
+ case SecProfSummary:
+ if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))
+ Flags.append("partial,");
+ break;
+ default:
+ break;
+ }
+ char &last = Flags.back();
+ if (last == ',')
+ last = '}';
+ else
+ Flags.append("}");
+ return Flags;
+}
+
bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
uint64_t TotalSecsSize = 0;
for (auto &Entry : SecHdrTable) {
OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset
- << ", Size: " << Entry.Size << "\n";
+ << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
+ << "\n";
+ ;
TotalSecsSize += getSectionSize(Entry.Type);
}
uint64_t HeaderSize = SecHdrTable.front().Offset;
diff --git a/llvm/test/tools/llvm-profdata/show-prof-info.test b/llvm/test/tools/llvm-profdata/show-prof-info.test
new file mode 100644
index 000000000000..ca91051dc1c8
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/show-prof-info.test
@@ -0,0 +1,12 @@
+REQUIRES: zlib
+; RUN: llvm-profdata merge -sample -extbinary -use-md5 -compress-all-sections -partial-profile -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile.proftext -o %t.1.output
+; RUN: wc -c < %t.1.output > %t.txt
+; RUN: llvm-profdata show -sample -show-sec-info-only %t.1.output >> %t.txt
+; RUN: FileCheck %s --input-file=%t.txt
+; CHECK: [[FILESIZE:.*]]
+; To check llvm-profdata shows the correct flags for ProfileSummarySection.
+; CHECK: ProfileSummarySection {{.*}} Flags: {compressed,partial}
+; To check llvm-profdata shows the correct flags for NameTableSection.
+; CHECK: NameTableSection {{.*}} Flags: {compressed,md5}
+; To check llvm-profdata shows the correct file size.
+; CHECK: [[FILESIZE]]
diff --git a/llvm/test/tools/llvm-profdata/show-prof-size.test b/llvm/test/tools/llvm-profdata/show-prof-size.test
deleted file mode 100644
index a77360c8a673..000000000000
--- a/llvm/test/tools/llvm-profdata/show-prof-size.test
+++ /dev/null
@@ -1,7 +0,0 @@
-; RUN: llvm-profdata merge -sample -extbinary -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile.proftext -o %t.1.output
-; RUN: wc -c < %t.1.output > %t.txt
-; RUN: llvm-profdata show -sample -show-sec-info-only %t.1.output >> %t.txt
-; RUN: FileCheck %s --input-file=%t.txt
-; Check llvm-profdata shows the correct file size.
-; CHECK: [[FILESIZE:.*]]
-; CHECK: [[FILESIZE]]
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 90ce8400c845..482bd1c842d1 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -448,7 +448,8 @@ static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
ProfileFormat OutputFormat,
MemoryBuffer *Buffer,
sampleprof::ProfileSymbolList &WriterList,
- bool CompressAllSections, bool UseMD5) {
+ bool CompressAllSections, bool UseMD5,
+ bool PartialProfile) {
populateProfileSymbolList(Buffer, WriterList);
if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary)
warn("Profile Symbol list is not empty but the output format is not "
@@ -468,13 +469,19 @@ static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
else
Writer.setUseMD5();
}
+ if (PartialProfile) {
+ if (OutputFormat != PF_Ext_Binary)
+ warn("-partial-profile is ignored. Specify -extbinary to enable it");
+ else
+ Writer.setPartialProfile();
+ }
}
static void
mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
StringRef OutputFilename, ProfileFormat OutputFormat,
StringRef ProfileSymbolListFile, bool CompressAllSections,
- bool UseMD5, FailureMode FailMode) {
+ bool UseMD5, bool PartialProfile, FailureMode FailMode) {
using namespace sampleprof;
StringMap<FunctionSamples> ProfileMap;
SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers;
@@ -531,7 +538,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
// Make sure Buffer lives as long as WriterList.
auto Buffer = getInputFileBuf(ProfileSymbolListFile);
handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList,
- CompressAllSections, UseMD5);
+ CompressAllSections, UseMD5, PartialProfile);
Writer->write(ProfileMap);
}
@@ -663,6 +670,10 @@ static int merge_main(int argc, const char *argv[]) {
"use-md5", cl::init(false), cl::Hidden,
cl::desc("Choose to use MD5 to represent string in name table (only "
"meaningful for -extbinary)"));
+ cl::opt<bool> PartialProfile(
+ "partial-profile", cl::init(false), cl::Hidden,
+ cl::desc("Set the profile to be a partial profile (only meaningful "
+ "for -extbinary)"));
cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n");
@@ -697,7 +708,7 @@ static int merge_main(int argc, const char *argv[]) {
else
mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename,
OutputFormat, ProfileSymbolListFile, CompressAllSections,
- UseMD5, FailureMode);
+ UseMD5, PartialProfile, FailureMode);
return 0;
}
More information about the llvm-commits
mailing list