[llvm] [ProfileData] Use SortedVectorMap for BodySampleMap, TypeCountMap, and CallsiteTypeMap (PR #216530)
Kazu Hirata via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 16 17:14:51 PDT 2026
https://github.com/kazutakahirata updated https://github.com/llvm/llvm-project/pull/216530
>From c679e5875f098130e6649964c0314cc81b3e9761 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Thu, 13 Aug 2026 21:43:24 -0700
Subject: [PATCH 1/2] [ProfileData] Use SortedVectorMap for BodySampleMap,
TypeCountMap, and CallsiteTypeMap
This patch switches several maps in the sample profile reader/writer
from std::map to SortedVectorMap.
The memory efficiency of SortedVectorMap improves multiple performance
metrics:
Profile merging:
Metric Baseline SortedVectorMap Change
-------------------------------------------------------
Wall Clock 200.22s 157.73s -21.2%
User Time 108.93s 92.79s -14.8%
System Time 70.34s 45.83s -34.8%
Total CPU Time 179.27s 138.62s -22.7%
Peak RSS 58.42 GiB 48.11 GiB -17.6%
Minor Page Faults 28,705,225 17,904,042 -37.6%
Major Page Faults 7 0 -100.0%
AutoFDO Compilation: I see small speedups randing between 0.04% to
0.10% among 100,000 invocations, depending on ThinLTO phases.
Assisted-by: Antigravity
---
llvm/include/llvm/ProfileData/SampleProf.h | 18 +++++++++++++++---
llvm/lib/ProfileData/SampleProf.cpp | 1 +
llvm/lib/ProfileData/SampleProfReader.cpp | 4 ++++
llvm/tools/llvm-profdata/llvm-profdata.cpp | 1 +
4 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index f439b704d897a..8e883ddc15b11 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -372,7 +372,7 @@ namespace sampleprof {
/// represents its counter.
/// TODO: The class name FunctionId should be renamed to SymbolId in a refactor
/// change.
-using TypeCountMap = std::map<FunctionId, uint64_t>;
+using TypeCountMap = SortedVectorMap<FunctionId, uint64_t, 0>;
/// Write \p Map to the output stream. Keys are linearized using \p NameTable
/// and written as ULEB128. Values are written as ULEB128 as well.
@@ -805,12 +805,12 @@ inline raw_ostream &operator<<(raw_ostream &OS, const SampleContext &Context) {
class FunctionSamples;
class SampleProfileReaderItaniumRemapper;
-using BodySampleMap = std::map<LineLocation, SampleRecord>;
+using BodySampleMap = SortedVectorMap<LineLocation, SampleRecord, 0>;
// NOTE: Using a StringMap here makes parsed profiles consume around 17% more
// memory, which is *very* significant for large profiles.
using FunctionSamplesMap = std::map<FunctionId, FunctionSamples>;
using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
-using CallsiteTypeMap = std::map<LineLocation, TypeCountMap>;
+using CallsiteTypeMap = SortedVectorMap<LineLocation, TypeCountMap, 0>;
using LocToLocMap = DenseMap<LineLocation, LineLocation>;
/// Representation of the samples collected for a function.
@@ -872,6 +872,14 @@ class FunctionSamples {
return BodySamples[Location].merge(SampleRecord, Weight);
}
+ void reserveBodySamples(size_t NumEntries) {
+ BodySamples.reserve(NumEntries);
+ }
+
+ void reserveCallsiteTypeCounts(size_t NumEntries) {
+ VirtualCallsiteTypeCounts.reserve(NumEntries);
+ }
+
// Remove a call target and decrease the body sample correspondingly. Return
// the number of body samples actually decreased.
uint64_t removeCalledTargetAndBodySample(uint32_t LineOffset,
@@ -1114,6 +1122,7 @@ class FunctionSamples {
"T must be a map with StringRef or FunctionId as key and "
"uint64_t as value");
TypeCountMap &TypeCounts = getTypeSamplesAt(Loc);
+ TypeCounts.reserve(Other.size());
bool Overflowed = false;
for (const auto &[Type, Count] : Other) {
@@ -1169,6 +1178,7 @@ class FunctionSamples {
addTotalSamples(Other.getTotalSamples(), Weight));
mergeSampleProfErrors(Result,
addHeadSamples(Other.getHeadSamples(), Weight));
+ BodySamples.reserve(Other.getBodySamples().size());
for (const auto &I : Other.getBodySamples()) {
const LineLocation &Loc = I.first;
const SampleRecord &Rec = I.second;
@@ -1181,6 +1191,7 @@ class FunctionSamples {
mergeSampleProfErrors(Result,
FSMap[Rec.first].merge(Rec.second, Weight));
}
+ VirtualCallsiteTypeCounts.reserve(Other.getCallsiteTypeCounts().size());
for (const auto &[Loc, OtherTypeMap] : Other.getCallsiteTypeCounts())
mergeSampleProfErrors(
Result, addCallsiteVTableTypeProfAt(Loc, OtherTypeMap, Weight));
@@ -1626,6 +1637,7 @@ class ProfileConverter {
// We recompute TotalSamples later, so here set to zero.
Profile.setTotalSamples(0);
} else {
+ Profile.reserveBodySamples(FS.getBodySamples().size());
for (const auto &[LineLocation, SampleRecord] : FS.getBodySamples()) {
Profile.addSampleRecord(LineLocation, SampleRecord);
}
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 22c7f8365ec38..bddf6f5b9f5b6 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -143,6 +143,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other,
uint64_t Weight) {
sampleprof_error Result;
Result = addSamples(Other.getSamples(), Weight);
+ CallTargets.reserve(Other.getCallTargets().size());
for (const auto &I : Other.getCallTargets()) {
mergeSampleProfErrors(Result, addCalledTarget(I.first, I.second, Weight));
}
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 8f50873c72d03..531d3e4b53a8f 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -660,6 +660,7 @@ SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
auto NumVTableTypes = readNumber<uint32_t>();
if (std::error_code EC = NumVTableTypes.getError())
return EC;
+ M.reserve(*NumVTableTypes);
for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
auto VTableType(readStringFromTable());
@@ -693,6 +694,7 @@ SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
auto NumCallsites = readNumber<uint32_t>();
if (std::error_code EC = NumCallsites.getError())
return EC;
+ FProfile.reserveCallsiteTypeCounts(*NumCallsites);
for (uint32_t I = 0; I < *NumCallsites; ++I) {
auto LineOffset = readNumber<uint64_t>();
@@ -727,6 +729,7 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
auto NumRecords = readNumber<uint32_t>();
if (std::error_code EC = NumRecords.getError())
return EC;
+ FProfile.reserveBodySamples(*NumRecords);
for (uint32_t I = 0; I < *NumRecords; ++I) {
auto LineOffset = readNumber<uint64_t>();
@@ -2012,6 +2015,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
}
FProfile->setFunction(FunctionId(Name));
+ FProfile->reserveBodySamples(NumPosCounts);
for (uint32_t I = 0; I < NumPosCounts; ++I) {
uint32_t Offset;
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index d3c83457a5d81..529c3590107c4 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -1492,6 +1492,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
Result.setFunction(Remapper(Samples.getFunction()));
Result.addTotalSamples(Samples.getTotalSamples());
Result.addHeadSamples(Samples.getHeadSamples());
+ Result.reserveBodySamples(Samples.getBodySamples().size());
for (const auto &BodySample : Samples.getBodySamples()) {
uint32_t MaskedDiscriminator =
BodySample.first.Discriminator & getDiscriminatorMask();
>From 54ada51174ba2396f659b4bb04c2e22b1309a02a Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Sun, 16 Aug 2026 17:06:36 -0700
Subject: [PATCH 2/2] Address comments.
---
llvm/include/llvm/ProfileData/SampleProf.h | 64 +++++++++++++++-------
1 file changed, 43 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 8e883ddc15b11..2165ed67da973 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -460,7 +460,12 @@ class SampleRecord {
bool hasCalls() const { return !CallTargets.empty(); }
uint64_t getSamples() const { return NumSamples; }
- const CallTargetMap &getCallTargets() const { return CallTargets; }
+ /// Return the call targets collected in this sample record.
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this SampleRecord.
+ const CallTargetMap &getCallTargets() const LLVM_LIFETIME_BOUND {
+ return CallTargets;
+ }
SortedCallTargetSet getSortedCallTargets() const {
return sortCallTargets(CallTargets);
}
@@ -975,8 +980,11 @@ class FunctionSamples {
/// Returns the call target map collected at a given location.
/// Each location is specified by \p LineOffset and \p Discriminator.
/// If the location is not found in profile, return error.
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
ErrorOr<const SampleRecord::CallTargetMap &>
- findCallTargetMapAt(uint32_t LineOffset, uint32_t Discriminator) const {
+ findCallTargetMapAt(uint32_t LineOffset,
+ uint32_t Discriminator) const LLVM_LIFETIME_BOUND {
const auto &Ret = BodySamples.find(
mapIRLocToProfileLoc(LineLocation(LineOffset, Discriminator)));
if (Ret == BodySamples.end())
@@ -986,8 +994,10 @@ class FunctionSamples {
/// Returns the call target map collected at a given location specified by \p
/// CallSite. If the location is not found in profile, return error.
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
ErrorOr<const SampleRecord::CallTargetMap &>
- findCallTargetMapAt(const LineLocation &CallSite) const {
+ findCallTargetMapAt(const LineLocation &CallSite) const LLVM_LIFETIME_BOUND {
const auto &Ret = BodySamples.find(mapIRLocToProfileLoc(CallSite));
if (Ret == BodySamples.end())
return std::error_code();
@@ -995,13 +1005,14 @@ class FunctionSamples {
}
/// Return the function samples at the given callsite location.
- FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) {
+ FunctionSamplesMap &
+ functionSamplesAt(const LineLocation &Loc) LLVM_LIFETIME_BOUND {
return CallsiteSamples[mapIRLocToProfileLoc(Loc)];
}
/// Returns the FunctionSamplesMap at the given \p Loc.
const FunctionSamplesMap *
- findFunctionSamplesMapAt(const LineLocation &Loc) const {
+ findFunctionSamplesMapAt(const LineLocation &Loc) const LLVM_LIFETIME_BOUND {
auto Iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc));
if (Iter == CallsiteSamples.end())
return nullptr;
@@ -1009,7 +1020,10 @@ class FunctionSamples {
}
/// Returns the TypeCountMap for inlined callsites at the given \p Loc.
- const TypeCountMap *findCallsiteTypeSamplesAt(const LineLocation &Loc) const {
+ /// The returned pointer may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
+ const TypeCountMap *
+ findCallsiteTypeSamplesAt(const LineLocation &Loc) const LLVM_LIFETIME_BOUND {
auto Iter = VirtualCallsiteTypeCounts.find(mapIRLocToProfileLoc(Loc));
if (Iter == VirtualCallsiteTypeCounts.end())
return nullptr;
@@ -1022,11 +1036,11 @@ class FunctionSamples {
/// \p Loc with the maximum total sample count. If \p Remapper or \p
/// FuncNameToProfNameMap is not nullptr, use them to find FunctionSamples
/// with equivalent name as \p CalleeName.
- LLVM_ABI const FunctionSamples *
- findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName,
- SampleProfileReaderItaniumRemapper *Remapper,
- const HashKeyMap<DenseMap, FunctionId, FunctionId>
- *FuncNameToProfNameMap = nullptr) const;
+ LLVM_ABI const FunctionSamples *findFunctionSamplesAt(
+ const LineLocation &Loc, StringRef CalleeName,
+ SampleProfileReaderItaniumRemapper *Remapper,
+ const HashKeyMap<DenseMap, FunctionId, FunctionId>
+ *FuncNameToProfNameMap = nullptr) const LLVM_LIFETIME_BOUND;
bool empty() const { return TotalSamples == 0; }
@@ -1070,10 +1084,14 @@ class FunctionSamples {
}
/// Return all the samples collected in the body of the function.
- const BodySampleMap &getBodySamples() const { return BodySamples; }
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
+ const BodySampleMap &getBodySamples() const LLVM_LIFETIME_BOUND {
+ return BodySamples;
+ }
/// Return all the callsite samples collected in the body of the function.
- const CallsiteSampleMap &getCallsiteSamples() const {
+ const CallsiteSampleMap &getCallsiteSamples() const LLVM_LIFETIME_BOUND {
return CallsiteSamples;
}
@@ -1082,14 +1100,18 @@ class FunctionSamples {
/// Returns vtable access samples for the C++ types collected in this
/// function.
- const CallsiteTypeMap &getCallsiteTypeCounts() const {
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
+ const CallsiteTypeMap &getCallsiteTypeCounts() const LLVM_LIFETIME_BOUND {
return VirtualCallsiteTypeCounts;
}
/// Returns the vtable access samples for the C++ types for \p Loc.
/// Under the hood, the caller-specified \p Loc will be un-drifted before the
/// type sample lookup if possible.
- TypeCountMap &getTypeSamplesAt(const LineLocation &Loc) {
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
+ TypeCountMap &getTypeSamplesAt(const LineLocation &Loc) LLVM_LIFETIME_BOUND {
return VirtualCallsiteTypeCounts[mapIRLocToProfileLoc(Loc)];
}
@@ -1360,13 +1382,13 @@ class FunctionSamples {
/// If \p Remapper or \p FuncNameToProfNameMap is not nullptr, it will be used
/// to find matching FunctionSamples with not exactly the same but equivalent
/// name.
- LLVM_ABI const FunctionSamples *
- findFunctionSamples(const DILocation *DIL,
- SampleProfileReaderItaniumRemapper *Remapper = nullptr,
- const HashKeyMap<DenseMap, FunctionId, FunctionId>
- *FuncNameToProfNameMap = nullptr) const;
+ LLVM_ABI const FunctionSamples *findFunctionSamples(
+ const DILocation *DIL,
+ SampleProfileReaderItaniumRemapper *Remapper = nullptr,
+ const HashKeyMap<DenseMap, FunctionId, FunctionId>
+ *FuncNameToProfNameMap = nullptr) const LLVM_LIFETIME_BOUND;
- SampleContext &getContext() const { return Context; }
+ SampleContext &getContext() const LLVM_LIFETIME_BOUND { return Context; }
void setContext(const SampleContext &FContext) { Context = FContext; }
More information about the llvm-commits
mailing list