[llvm] 8f8fea3 - [ProfileData] Use SortedVectorMap for BodySampleMap, TypeCountMap, and CallsiteTypeMap (#216530)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 22 15:40:27 PDT 2026
Author: Kazu Hirata
Date: 2026-08-22T15:40:22-07:00
New Revision: 8f8fea3d74bfe024c85236c3f87b46e0ef710d77
URL: https://github.com/llvm/llvm-project/commit/8f8fea3d74bfe024c85236c3f87b46e0ef710d77
DIFF: https://github.com/llvm/llvm-project/commit/8f8fea3d74bfe024c85236c3f87b46e0ef710d77.diff
LOG: [ProfileData] Use SortedVectorMap for BodySampleMap, TypeCountMap, and CallsiteTypeMap (#216530)
This patch switches several maps in the sample profile reader/writer
from std::map to SortedVectorMap.
The memory efficiency of SortedVectorMap improves multiple performance
metrics:
Profile merging:
Metric Baseline SortedVectorMap Change
-------------------------------------------------------
Wall Clock 200.22s 157.73s -21.2%
User Time 108.93s 92.79s -14.8%
System Time 70.34s 45.83s -34.8%
Total CPU Time 179.27s 138.62s -22.7%
Peak RSS 58.42 GiB 48.11 GiB -17.6%
Minor Page Faults 28,705,225 17,904,042 -37.6%
Major Page Faults 7 0 -100.0%
AutoFDO Compilation: I see small speedups randing between 0.04% to
0.10% among 100,000 invocations, depending on ThinLTO phases.
Assisted-by: Antigravity
Added:
Modified:
llvm/include/llvm/ProfileData/SampleProf.h
llvm/lib/ProfileData/SampleProf.cpp
llvm/lib/ProfileData/SampleProfReader.cpp
llvm/tools/llvm-profdata/llvm-profdata.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index f439b704d897a..68bdc0d937956 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -372,7 +372,7 @@ namespace sampleprof {
/// represents its counter.
/// TODO: The class name FunctionId should be renamed to SymbolId in a refactor
/// change.
-using TypeCountMap = std::map<FunctionId, uint64_t>;
+using TypeCountMap = SortedVectorMap<FunctionId, uint64_t, 0>;
/// Write \p Map to the output stream. Keys are linearized using \p NameTable
/// and written as ULEB128. Values are written as ULEB128 as well.
@@ -460,7 +460,12 @@ class SampleRecord {
bool hasCalls() const { return !CallTargets.empty(); }
uint64_t getSamples() const { return NumSamples; }
- const CallTargetMap &getCallTargets() const { return CallTargets; }
+ /// Return the call targets collected in this sample record.
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this SampleRecord.
+ const CallTargetMap &getCallTargets() const LLVM_LIFETIME_BOUND {
+ return CallTargets;
+ }
SortedCallTargetSet getSortedCallTargets() const {
return sortCallTargets(CallTargets);
}
@@ -805,12 +810,12 @@ inline raw_ostream &operator<<(raw_ostream &OS, const SampleContext &Context) {
class FunctionSamples;
class SampleProfileReaderItaniumRemapper;
-using BodySampleMap = std::map<LineLocation, SampleRecord>;
+using BodySampleMap = SortedVectorMap<LineLocation, SampleRecord, 0>;
// NOTE: Using a StringMap here makes parsed profiles consume around 17% more
// memory, which is *very* significant for large profiles.
using FunctionSamplesMap = std::map<FunctionId, FunctionSamples>;
using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
-using CallsiteTypeMap = std::map<LineLocation, TypeCountMap>;
+using CallsiteTypeMap = SortedVectorMap<LineLocation, TypeCountMap, 0>;
using LocToLocMap = DenseMap<LineLocation, LineLocation>;
/// Representation of the samples collected for a function.
@@ -872,6 +877,14 @@ class FunctionSamples {
return BodySamples[Location].merge(SampleRecord, Weight);
}
+ void reserveBodySamples(size_t NumEntries) {
+ BodySamples.reserve(NumEntries);
+ }
+
+ void reserveCallsiteTypeCounts(size_t NumEntries) {
+ VirtualCallsiteTypeCounts.reserve(NumEntries);
+ }
+
// Remove a call target and decrease the body sample correspondingly. Return
// the number of body samples actually decreased.
uint64_t removeCalledTargetAndBodySample(uint32_t LineOffset,
@@ -967,8 +980,11 @@ class FunctionSamples {
/// Returns the call target map collected at a given location.
/// Each location is specified by \p LineOffset and \p Discriminator.
/// If the location is not found in profile, return error.
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
ErrorOr<const SampleRecord::CallTargetMap &>
- findCallTargetMapAt(uint32_t LineOffset, uint32_t Discriminator) const {
+ findCallTargetMapAt(uint32_t LineOffset,
+ uint32_t Discriminator) const LLVM_LIFETIME_BOUND {
const auto &Ret = BodySamples.find(
mapIRLocToProfileLoc(LineLocation(LineOffset, Discriminator)));
if (Ret == BodySamples.end())
@@ -978,8 +994,10 @@ class FunctionSamples {
/// Returns the call target map collected at a given location specified by \p
/// CallSite. If the location is not found in profile, return error.
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
ErrorOr<const SampleRecord::CallTargetMap &>
- findCallTargetMapAt(const LineLocation &CallSite) const {
+ findCallTargetMapAt(const LineLocation &CallSite) const LLVM_LIFETIME_BOUND {
const auto &Ret = BodySamples.find(mapIRLocToProfileLoc(CallSite));
if (Ret == BodySamples.end())
return std::error_code();
@@ -987,13 +1005,14 @@ class FunctionSamples {
}
/// Return the function samples at the given callsite location.
- FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) {
+ FunctionSamplesMap &
+ functionSamplesAt(const LineLocation &Loc) LLVM_LIFETIME_BOUND {
return CallsiteSamples[mapIRLocToProfileLoc(Loc)];
}
/// Returns the FunctionSamplesMap at the given \p Loc.
const FunctionSamplesMap *
- findFunctionSamplesMapAt(const LineLocation &Loc) const {
+ findFunctionSamplesMapAt(const LineLocation &Loc) const LLVM_LIFETIME_BOUND {
auto Iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc));
if (Iter == CallsiteSamples.end())
return nullptr;
@@ -1001,7 +1020,10 @@ class FunctionSamples {
}
/// Returns the TypeCountMap for inlined callsites at the given \p Loc.
- const TypeCountMap *findCallsiteTypeSamplesAt(const LineLocation &Loc) const {
+ /// The returned pointer may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
+ const TypeCountMap *
+ findCallsiteTypeSamplesAt(const LineLocation &Loc) const LLVM_LIFETIME_BOUND {
auto Iter = VirtualCallsiteTypeCounts.find(mapIRLocToProfileLoc(Loc));
if (Iter == VirtualCallsiteTypeCounts.end())
return nullptr;
@@ -1014,11 +1036,11 @@ class FunctionSamples {
/// \p Loc with the maximum total sample count. If \p Remapper or \p
/// FuncNameToProfNameMap is not nullptr, use them to find FunctionSamples
/// with equivalent name as \p CalleeName.
- LLVM_ABI const FunctionSamples *
- findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName,
- SampleProfileReaderItaniumRemapper *Remapper,
- const HashKeyMap<DenseMap, FunctionId, FunctionId>
- *FuncNameToProfNameMap = nullptr) const;
+ LLVM_ABI const FunctionSamples *findFunctionSamplesAt(
+ const LineLocation &Loc, StringRef CalleeName,
+ SampleProfileReaderItaniumRemapper *Remapper,
+ const HashKeyMap<DenseMap, FunctionId, FunctionId>
+ *FuncNameToProfNameMap = nullptr) const LLVM_LIFETIME_BOUND;
bool empty() const { return TotalSamples == 0; }
@@ -1062,10 +1084,14 @@ class FunctionSamples {
}
/// Return all the samples collected in the body of the function.
- const BodySampleMap &getBodySamples() const { return BodySamples; }
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
+ const BodySampleMap &getBodySamples() const LLVM_LIFETIME_BOUND {
+ return BodySamples;
+ }
/// Return all the callsite samples collected in the body of the function.
- const CallsiteSampleMap &getCallsiteSamples() const {
+ const CallsiteSampleMap &getCallsiteSamples() const LLVM_LIFETIME_BOUND {
return CallsiteSamples;
}
@@ -1074,14 +1100,18 @@ class FunctionSamples {
/// Returns vtable access samples for the C++ types collected in this
/// function.
- const CallsiteTypeMap &getCallsiteTypeCounts() const {
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
+ const CallsiteTypeMap &getCallsiteTypeCounts() const LLVM_LIFETIME_BOUND {
return VirtualCallsiteTypeCounts;
}
/// Returns the vtable access samples for the C++ types for \p Loc.
/// Under the hood, the caller-specified \p Loc will be un-drifted before the
/// type sample lookup if possible.
- TypeCountMap &getTypeSamplesAt(const LineLocation &Loc) {
+ /// The returned reference may be invalidated by subsequent modifications to
+ /// this FunctionSamples.
+ TypeCountMap &getTypeSamplesAt(const LineLocation &Loc) LLVM_LIFETIME_BOUND {
return VirtualCallsiteTypeCounts[mapIRLocToProfileLoc(Loc)];
}
@@ -1114,6 +1144,7 @@ class FunctionSamples {
"T must be a map with StringRef or FunctionId as key and "
"uint64_t as value");
TypeCountMap &TypeCounts = getTypeSamplesAt(Loc);
+ TypeCounts.reserve(TypeCounts.size() + Other.size());
bool Overflowed = false;
for (const auto &[Type, Count] : Other) {
@@ -1169,6 +1200,7 @@ class FunctionSamples {
addTotalSamples(Other.getTotalSamples(), Weight));
mergeSampleProfErrors(Result,
addHeadSamples(Other.getHeadSamples(), Weight));
+ BodySamples.reserve(BodySamples.size() + Other.getBodySamples().size());
for (const auto &I : Other.getBodySamples()) {
const LineLocation &Loc = I.first;
const SampleRecord &Rec = I.second;
@@ -1181,6 +1213,8 @@ class FunctionSamples {
mergeSampleProfErrors(Result,
FSMap[Rec.first].merge(Rec.second, Weight));
}
+ VirtualCallsiteTypeCounts.reserve(VirtualCallsiteTypeCounts.size() +
+ Other.getCallsiteTypeCounts().size());
for (const auto &[Loc, OtherTypeMap] : Other.getCallsiteTypeCounts())
mergeSampleProfErrors(
Result, addCallsiteVTableTypeProfAt(Loc, OtherTypeMap, Weight));
@@ -1349,13 +1383,13 @@ class FunctionSamples {
/// If \p Remapper or \p FuncNameToProfNameMap is not nullptr, it will be used
/// to find matching FunctionSamples with not exactly the same but equivalent
/// name.
- LLVM_ABI const FunctionSamples *
- findFunctionSamples(const DILocation *DIL,
- SampleProfileReaderItaniumRemapper *Remapper = nullptr,
- const HashKeyMap<DenseMap, FunctionId, FunctionId>
- *FuncNameToProfNameMap = nullptr) const;
+ LLVM_ABI const FunctionSamples *findFunctionSamples(
+ const DILocation *DIL,
+ SampleProfileReaderItaniumRemapper *Remapper = nullptr,
+ const HashKeyMap<DenseMap, FunctionId, FunctionId>
+ *FuncNameToProfNameMap = nullptr) const LLVM_LIFETIME_BOUND;
- SampleContext &getContext() const { return Context; }
+ SampleContext &getContext() const LLVM_LIFETIME_BOUND { return Context; }
void setContext(const SampleContext &FContext) { Context = FContext; }
@@ -1626,6 +1660,7 @@ class ProfileConverter {
// We recompute TotalSamples later, so here set to zero.
Profile.setTotalSamples(0);
} else {
+ Profile.reserveBodySamples(FS.getBodySamples().size());
for (const auto &[LineLocation, SampleRecord] : FS.getBodySamples()) {
Profile.addSampleRecord(LineLocation, SampleRecord);
}
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 22c7f8365ec38..5e090d6e5c765 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -143,6 +143,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other,
uint64_t Weight) {
sampleprof_error Result;
Result = addSamples(Other.getSamples(), Weight);
+ CallTargets.reserve(CallTargets.size() + Other.getCallTargets().size());
for (const auto &I : Other.getCallTargets()) {
mergeSampleProfErrors(Result, addCalledTarget(I.first, I.second, Weight));
}
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 8f50873c72d03..531d3e4b53a8f 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -660,6 +660,7 @@ SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
auto NumVTableTypes = readNumber<uint32_t>();
if (std::error_code EC = NumVTableTypes.getError())
return EC;
+ M.reserve(*NumVTableTypes);
for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
auto VTableType(readStringFromTable());
@@ -693,6 +694,7 @@ SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
auto NumCallsites = readNumber<uint32_t>();
if (std::error_code EC = NumCallsites.getError())
return EC;
+ FProfile.reserveCallsiteTypeCounts(*NumCallsites);
for (uint32_t I = 0; I < *NumCallsites; ++I) {
auto LineOffset = readNumber<uint64_t>();
@@ -727,6 +729,7 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
auto NumRecords = readNumber<uint32_t>();
if (std::error_code EC = NumRecords.getError())
return EC;
+ FProfile.reserveBodySamples(*NumRecords);
for (uint32_t I = 0; I < *NumRecords; ++I) {
auto LineOffset = readNumber<uint64_t>();
@@ -2012,6 +2015,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
}
FProfile->setFunction(FunctionId(Name));
+ FProfile->reserveBodySamples(NumPosCounts);
for (uint32_t I = 0; I < NumPosCounts; ++I) {
uint32_t Offset;
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index fa4a234296cea..49c645598f475 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -1502,6 +1502,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
Result.setFunction(Remapper(Samples.getFunction()));
Result.addTotalSamples(Samples.getTotalSamples());
Result.addHeadSamples(Samples.getHeadSamples());
+ Result.reserveBodySamples(Samples.getBodySamples().size());
for (const auto &BodySample : Samples.getBodySamples()) {
uint32_t MaskedDiscriminator =
BodySample.first.Discriminator & getDiscriminatorMask();
More information about the llvm-commits
mailing list