[llvm] [ProfileData] Support merging MD5-based ProfileSymbolList (PR #226594)
Kazu Hirata via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 26 02:55:49 PDT 2026
https://github.com/kazutakahirata updated https://github.com/llvm/llvm-project/pull/226594
>From e417abcc468378c51b74dd0fecabde1086b03931 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Fri, 25 Sep 2026 11:21:11 -0700
Subject: [PATCH 1/2] [ProfileData] Support merging MD5-based ProfileSymbolList
This patch supports merging MD5-based ProfileSymbolList instances and
writing the merged result to an extensible binary profile.
#210235 introduced the MD5-based ProfileSymbolList section in the
Eytzinger layout, where profile merging was initially supported only
from strings to an MD5-based Eytzinger array.
This patch adds DenseSet<uint64_t> GUIDs to ProfileSymbolList to
accumulate 64-bit MD5 hashes when merging MD5-based symbol lists,
while keeping ColdGUIDTable (EytzingerTableSpan) for zero-copy lookups
during compilation. collectGUIDs, contains, and size are updated to
query GUIDs when populated.
RFC:
https://discourse.llvm.org/t/rfc-faster-sample-profile-loading/90957/7
Assisted-by: Antigravity
---
llvm/include/llvm/ProfileData/SampleProf.h | 59 ++++++++++++++-----
llvm/lib/ProfileData/SampleProfWriter.cpp | 3 -
.../llvm-profdata/profile-symbol-list.test | 15 +++--
llvm/unittests/ProfileData/SampleProfTest.cpp | 38 ++++++++++++
4 files changed, 94 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 6538f031b1f953..110f2c97a55d7e 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -1752,6 +1752,7 @@ class ProfileSymbolList {
/// copy indicates whether we need to copy the underlying memory
/// for the input Name.
void add(StringRef Name, bool Copy = false) {
+ assert(!IsMD5 && "Adding string to MD5 ProfileSymbolList is not supported");
if (!Copy) {
Syms.insert(Name);
return;
@@ -1760,39 +1761,61 @@ class ProfileSymbolList {
}
bool contains(StringRef Name) const {
- return IsMD5 ? ColdGUIDTable.contains(llvm::MD5Hash(Name))
- : Syms.count(Name);
+ if (!IsMD5)
+ return Syms.contains(Name);
+ uint64_t GUID = llvm::MD5Hash(Name);
+ return !ColdGUIDTable.empty() ? ColdGUIDTable.contains(GUID)
+ : GUIDs.contains(GUID);
}
void merge(const ProfileSymbolList &List) {
- assert(!List.IsMD5 &&
- "Merging pre-hashed MD5 ProfileSymbolList not yet implemented");
- for (auto Sym : List.Syms)
- add(Sym, true);
+ if (List.size() == 0)
+ return;
+ if (!List.IsMD5) {
+ assert(!IsMD5 &&
+ "Merging string and MD5 ProfileSymbolLists is not supported");
+ for (auto Sym : List.Syms)
+ add(Sym, true);
+ return;
+ }
+ assert(Syms.empty() && ColdGUIDTable.empty() &&
+ "Merging into non-empty string or ColdGUIDTable ProfileSymbolList "
+ "is not supported");
+ IsMD5 = true;
+ GUIDs.insert_range(List.ColdGUIDTable);
+ GUIDs.insert_range(List.GUIDs);
}
- unsigned size() const { return IsMD5 ? ColdGUIDTable.size() : Syms.size(); }
+ unsigned size() const {
+ if (!IsMD5)
+ return Syms.size();
+ return !ColdGUIDTable.empty() ? ColdGUIDTable.size() : GUIDs.size();
+ }
void reserve(size_t Size) { Syms.reserve(Size); }
std::vector<uint64_t> collectGUIDs() const {
- assert(!IsMD5 &&
- "Collecting GUIDs from existing MD5 table not yet implemented");
std::vector<uint64_t> Keys;
- Keys.reserve(Syms.size());
- llvm::append_range(Keys, llvm::map_range(Syms, llvm::MD5Hash));
+ Keys.reserve(size());
+ if (!IsMD5)
+ llvm::append_range(Keys, llvm::map_range(Syms, llvm::MD5Hash));
+ else if (!ColdGUIDTable.empty())
+ llvm::append_range(Keys, ColdGUIDTable);
+ else
+ llvm::append_range(Keys, GUIDs);
llvm::sort(Keys);
Keys.erase(llvm::unique(Keys), Keys.end());
return Keys;
}
void setColdGUIDTable(EytzingerTableSpan<support::ulittle64_t> Table) {
- assert(Syms.empty() &&
- "Setting ColdGUIDTable shadows existing strings in Syms");
+ assert(Syms.empty() && GUIDs.empty() &&
+ "Setting ColdGUIDTable shadows existing entries");
ColdGUIDTable = Table;
IsMD5 = true;
}
EytzingerTableSpan<support::ulittle64_t> getColdGUIDTable() const {
- assert(IsMD5 && "Retrieving ColdGUIDTable from non-MD5 ProfileSymbolList");
+ assert(IsMD5 && GUIDs.empty() &&
+ "Retrieving ColdGUIDTable from non-table-backed ProfileSymbolList");
return ColdGUIDTable;
}
bool isMD5() const { return IsMD5; }
@@ -1802,9 +1825,17 @@ class ProfileSymbolList {
LLVM_ABI void dump(raw_ostream &OS = dbgs()) const;
private:
+ // Whether symbols are stored as 64-bit MD5 hashes (in ColdGUIDTable or
+ // GUIDs) rather than plain strings (in Syms). At most one of Syms,
+ // ColdGUIDTable, or GUIDs is non-empty at any given time.
bool IsMD5 = false;
+ // Symbol names for string-based symbol lists (!IsMD5).
DenseSet<StringRef> Syms;
+ // Non-owning view of Eytzinger-ordered MD5 hashes backed by the profile
+ // reader's buffer, used for zero-copy lookups during compilation.
EytzingerTableSpan<support::ulittle64_t> ColdGUIDTable;
+ // Owning set of MD5 hashes populated when merging MD5 symbol lists.
+ DenseSet<uint64_t> GUIDs;
BumpPtrAllocator Allocator;
};
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index cae31792c9b353..d6a6aef1c59f92 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -617,9 +617,6 @@ std::error_code
SampleProfileWriterExtBinaryBase::writeMD5ProfileSymbolListSection() {
if (!ProfSymList || ProfSymList->size() == 0)
return sampleprof_error::success;
- assert(!ProfSymList->isMD5() &&
- "Writing MD5 ProfileSymbolListSection from existing MD5 "
- "table not yet implemented");
auto &OS = *OutputStream;
std::vector<uint64_t> Keys = ProfSymList->collectGUIDs();
diff --git a/llvm/test/tools/llvm-profdata/profile-symbol-list.test b/llvm/test/tools/llvm-profdata/profile-symbol-list.test
index e19fd044a02f38..23d818382aadd4 100644
--- a/llvm/test/tools/llvm-profdata/profile-symbol-list.test
+++ b/llvm/test/tools/llvm-profdata/profile-symbol-list.test
@@ -8,11 +8,18 @@
; NOSYMLIST: ProfileSymbolListSection {{.*}} Size: 0
-;; Verify that -md5-prof-sym-list records the md5 section flag for ProfileSymbolListSection.
-; RUN: llvm-profdata merge -sample -extbinary -md5-prof-sym-list -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile.proftext -o %t.md5.output
-; RUN: llvm-profdata show -sample -show-sec-info-only %t.md5.output | FileCheck %s -check-prefix=MD5
+;; Verify that -md5-prof-sym-list records the md5 section flag for ProfileSymbolListSection
+;; and that profiles with MD5 ProfileSymbolListSection can be merged.
+; RUN: llvm-profdata merge -sample -extbinary -md5-prof-sym-list -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile.proftext -o %t.md5.1.output
+; RUN: llvm-profdata show -sample -show-sec-info-only %t.md5.1.output | FileCheck %s -check-prefix=MD5-1
+; RUN: llvm-profdata merge -sample -extbinary -md5-prof-sym-list -prof-sym-list=%S/Inputs/profile-symbol-list-2.text %S/Inputs/sample-profile.proftext -o %t.md5.2.output
+; RUN: llvm-profdata show -sample -show-sec-info-only %t.md5.2.output | FileCheck %s -check-prefix=MD5-2
+; RUN: llvm-profdata merge -sample -extbinary -md5-prof-sym-list %t.md5.1.output %t.md5.2.output -o %t.md5.merged.output
+; RUN: llvm-profdata show -sample -show-sec-info-only %t.md5.merged.output | FileCheck %s -check-prefix=MD5-MERGED
-; MD5: ProfileSymbolListSection - Offset: {{.*}}, Size: {{.*}}, Flags: {{{.*}}md5}
+; MD5-1: ProfileSymbolListSection - Offset: {{.*}}, Size: 40, Flags: {{{.*}}md5}
+; MD5-2: ProfileSymbolListSection - Offset: {{.*}}, Size: 32, Flags: {{{.*}}md5}
+; MD5-MERGED: ProfileSymbolListSection - Offset: {{.*}}, Size: 72, Flags: {{{.*}}md5}
;; Generate two SampleFDO binary profiles and merge them.
;; Tests that the vtable counters in the merged profile are the aggregated
diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp
index 799f00094cbda8..a991333ef2bcf3 100644
--- a/llvm/unittests/ProfileData/SampleProfTest.cpp
+++ b/llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -1156,6 +1156,44 @@ TEST_F(SampleProfTest, ProfileSymbolListMD5) {
EXPECT_EQ(2u, List.size());
}
+TEST_F(SampleProfTest, ProfileSymbolListMD5Merge) {
+ auto Table1 = llvm::EytzingerTable<support::ulittle64_t>::create(
+ std::vector<uint64_t>{llvm::MD5Hash("foo"), llvm::MD5Hash("bar")});
+ ProfileSymbolList List1;
+ List1.setColdGUIDTable(Table1.asSpan());
+
+ auto Table2 = llvm::EytzingerTable<support::ulittle64_t>::create(
+ std::vector<uint64_t>{llvm::MD5Hash("bar"), llvm::MD5Hash("baz")});
+ ProfileSymbolList List2;
+ List2.setColdGUIDTable(Table2.asSpan());
+
+ // Merge two ColdGUIDTable-backed MD5 lists with an overlapping entry ("bar"),
+ // as well as default-constructed empty lists before and after.
+ ProfileSymbolList EmptyList;
+ ProfileSymbolList Merged;
+ Merged.merge(EmptyList);
+ Merged.merge(List1);
+ Merged.merge(EmptyList);
+ Merged.merge(List2);
+
+ EXPECT_TRUE(Merged.isMD5());
+ EXPECT_EQ(3u, Merged.size());
+ EXPECT_TRUE(Merged.contains("foo"));
+ EXPECT_TRUE(Merged.contains("bar"));
+ EXPECT_TRUE(Merged.contains("baz"));
+ EXPECT_FALSE(Merged.contains("qux"));
+
+ std::vector<uint64_t> ExpectedGUIDs = {
+ llvm::MD5Hash("foo"), llvm::MD5Hash("bar"), llvm::MD5Hash("baz")};
+ llvm::sort(ExpectedGUIDs);
+ EXPECT_EQ(ExpectedGUIDs, Merged.collectGUIDs());
+
+ // Transitive merge from a list whose entries are already in GUIDs.
+ ProfileSymbolList Transitive;
+ Transitive.merge(Merged);
+ EXPECT_EQ(ExpectedGUIDs, Transitive.collectGUIDs());
+}
+
struct ScopedHasUniqSuffix {
bool OldVal;
ScopedHasUniqSuffix(bool NewVal) : OldVal(FunctionSamples::HasUniqSuffix) {
>From 1a49bdce10a6c8fb1fc92006c678e5ed0b8e9ce2 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Sat, 26 Sep 2026 02:55:28 -0700
Subject: [PATCH 2/2] Address comments.
---
llvm/include/llvm/ProfileData/SampleProfWriter.h | 4 +++-
llvm/test/tools/llvm-profdata/profile-symbol-list.test | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 2f0bf68c8baff3..b785d73020de0b 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -331,7 +331,9 @@ class LLVM_ABI SampleProfileWriterExtBinaryBase
void setProfileSymbolList(ProfileSymbolList *PSL) override {
ProfSymList = PSL;
- };
+ if (ProfSymList && ProfSymList->isMD5())
+ setUseMD5ProfileSymbolList();
+ }
void setUseCtxSplitLayout() override {
resetSecLayout(SectionLayout::CtxSplitLayout);
diff --git a/llvm/test/tools/llvm-profdata/profile-symbol-list.test b/llvm/test/tools/llvm-profdata/profile-symbol-list.test
index 23d818382aadd4..3c90b0faf2402b 100644
--- a/llvm/test/tools/llvm-profdata/profile-symbol-list.test
+++ b/llvm/test/tools/llvm-profdata/profile-symbol-list.test
@@ -14,7 +14,7 @@
; RUN: llvm-profdata show -sample -show-sec-info-only %t.md5.1.output | FileCheck %s -check-prefix=MD5-1
; RUN: llvm-profdata merge -sample -extbinary -md5-prof-sym-list -prof-sym-list=%S/Inputs/profile-symbol-list-2.text %S/Inputs/sample-profile.proftext -o %t.md5.2.output
; RUN: llvm-profdata show -sample -show-sec-info-only %t.md5.2.output | FileCheck %s -check-prefix=MD5-2
-; RUN: llvm-profdata merge -sample -extbinary -md5-prof-sym-list %t.md5.1.output %t.md5.2.output -o %t.md5.merged.output
+; RUN: llvm-profdata merge -sample -extbinary %t.md5.1.output %t.md5.2.output -o %t.md5.merged.output
; RUN: llvm-profdata show -sample -show-sec-info-only %t.md5.merged.output | FileCheck %s -check-prefix=MD5-MERGED
; MD5-1: ProfileSymbolListSection - Offset: {{.*}}, Size: 40, Flags: {{{.*}}md5}
More information about the llvm-commits
mailing list