[llvm] eba5749 - [CSSPGO][llvm-profgen] Reimplement CS profile generator using context trie
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 27 23:30:07 PDT 2022
Author: wlei
Date: 2022-06-27T23:22:21-07:00
New Revision: eba5749262d9f1c6754984034c7a81fcd9bc3de6
URL: https://github.com/llvm/llvm-project/commit/eba5749262d9f1c6754984034c7a81fcd9bc3de6
DIFF: https://github.com/llvm/llvm-project/commit/eba5749262d9f1c6754984034c7a81fcd9bc3de6.diff
LOG: [CSSPGO][llvm-profgen] Reimplement CS profile generator using context trie
Our investigation showed ProfileMap's key is the bottleneck of the memory consumption for CS profile generation on some large services. This patch tries to optimize it by storing the CS function samples using the context trie tree structure instead of the context frame array ref. Parts of code in `ContextTrieNode` are reused.
Our experiment on one internal service showed that the context key's memory can be reduced from 80GB to 300MB.
To be compatible with non-CS profiles, the profile writer still needs to use ProfileMap as input, so rebuild the ProfileMap using the context trie in `postProcessProfiles`.
The optimization is not complete yet, next step is to reimplement Pre-inliner or profile trimmer, after that, ProfileMap should be small to be written.
Reviewed By: hoy, wenlei
Differential Revision: https://reviews.llvm.org/D125246
Added:
Modified:
llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
llvm/tools/llvm-profgen/ProfileGenerator.cpp
llvm/tools/llvm-profgen/ProfileGenerator.h
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
index 0e23c9f6bab71..45dec6242f2c2 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -105,6 +105,7 @@ class SampleContextTracker {
// deterministically.
using ContextSamplesTy = std::set<FunctionSamples *, ProfileComparer>;
+ SampleContextTracker() = default;
SampleContextTracker(SampleProfileMap &Profiles,
const DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap);
// Query context profile for a specific callee with given name at a given
@@ -122,6 +123,8 @@ class SampleContextTracker {
// Get all context profile for given function.
ContextSamplesTy &getAllContextSamplesFor(const Function &Func);
ContextSamplesTy &getAllContextSamplesFor(StringRef Name);
+ ContextTrieNode *getOrCreateContextPath(const SampleContext &Context,
+ bool AllowCreate);
// Query base profile for a given function. A base profile is a merged view
// of all context profiles for contexts that are not inlined.
FunctionSamples *getBaseSamplesFor(const Function &Func,
@@ -146,8 +149,6 @@ class SampleContextTracker {
ContextTrieNode *getContextFor(const DILocation *DIL);
ContextTrieNode *getCalleeContextFor(const DILocation *DIL,
StringRef CalleeName);
- ContextTrieNode *getOrCreateContextPath(const SampleContext &Context,
- bool AllowCreate);
ContextTrieNode *getTopLevelContextNode(StringRef FName);
ContextTrieNode &addTopLevelContextNode(StringRef FName);
ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo);
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index a7c418154758f..4bc72f7678216 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -710,35 +710,38 @@ void ProfileGeneratorBase::calculateAndShowDensity(
showDensitySuggestion(Density);
}
-FunctionSamples &CSProfileGenerator::getFunctionProfileForContext(
- const SampleContextFrameVector &Context, bool WasLeafInlined) {
- auto I = ProfileMap.find(SampleContext(Context));
- if (I == ProfileMap.end()) {
- // Save the new context for future references.
- SampleContextFrames NewContext = *Contexts.insert(Context).first;
- SampleContext FContext(NewContext, RawContext);
- auto Ret = ProfileMap.emplace(FContext, FunctionSamples());
- if (WasLeafInlined)
- FContext.setAttribute(ContextWasInlined);
- FunctionSamples &FProfile = Ret.first->second;
- FProfile.setContext(FContext);
- return Ret.first->second;
- } else {
- // Update ContextWasInlined attribute for existing contexts.
- // The current function can be called in two ways:
- // - when processing a probe of the current frame
- // - when processing the entry probe of an inlinee's frame, which
- // is then used to update the callsite count of the current frame.
- // The two can happen in any order, hence here we are making sure
- // `ContextWasInlined` is always set as expected.
- // TODO: Note that the former does not always happen if no probes of the
- // current frame has samples, and if the latter happens, we could lose the
- // attribute. This should be fixed.
- if (WasLeafInlined)
- I->second.getContext().setAttribute(ContextWasInlined);
+FunctionSamples *
+CSProfileGenerator::getOrCreateFunctionSamples(ContextTrieNode *ContextNode,
+ bool WasLeafInlined) {
+ FunctionSamples *FProfile = ContextNode->getFunctionSamples();
+ if (!FProfile) {
+ FSamplesList.emplace_back();
+ FProfile = &FSamplesList.back();
+ FProfile->setName(ContextNode->getFuncName());
+ ContextNode->setFunctionSamples(FProfile);
}
+ // Update ContextWasInlined attribute for existing contexts.
+ // The current function can be called in two ways:
+ // - when processing a probe of the current frame
+ // - when processing the entry probe of an inlinee's frame, which
+ // is then used to update the callsite count of the current frame.
+ // The two can happen in any order, hence here we are making sure
+ // `ContextWasInlined` is always set as expected.
+ // TODO: Note that the former does not always happen if no probes of the
+ // current frame has samples, and if the latter happens, we could lose the
+ // attribute. This should be fixed.
+ if (WasLeafInlined)
+ FProfile->getContext().setAttribute(ContextWasInlined);
+ return FProfile;
+}
- return I->second;
+ContextTrieNode *
+CSProfileGenerator::getOrCreateContextNode(const SampleContextFrames Context,
+ bool WasLeafInlined) {
+ ContextTrieNode *ContextNode =
+ ContextTracker.getOrCreateContextPath(Context, true);
+ getOrCreateFunctionSamples(ContextNode, WasLeafInlined);
+ return ContextNode;
}
void CSProfileGenerator::generateProfile() {
@@ -772,29 +775,49 @@ void CSProfileGenerator::computeSizeForProfiledFunctions() {
Binary->flushSymbolizer();
}
+void CSProfileGenerator::updateFunctionSamples() {
+ std::queue<ContextTrieNode *> NodeQueue;
+ NodeQueue.push(&getRootContext());
+
+ while (!NodeQueue.empty()) {
+ ContextTrieNode *Node = NodeQueue.front();
+ NodeQueue.pop();
+
+ FunctionSamples *FSamples = Node->getFunctionSamples();
+ if (FSamples) {
+ if (UpdateTotalSamples)
+ FSamples->updateTotalSamples();
+ FSamples->updateCallsiteSamples();
+ }
+
+ for (auto &It : Node->getAllChildContext())
+ NodeQueue.push(&It.second);
+ }
+}
+
void CSProfileGenerator::generateLineNumBasedProfile() {
for (const auto &CI : *SampleCounters) {
const auto *CtxKey = cast<StringBasedCtxKey>(CI.first.getPtr());
- FunctionSamples *FunctionProfile = nullptr;
+ ContextTrieNode *ContextNode = &getRootContext();
// Sample context will be empty if the jump is an external-to-internal call
// pattern, the head samples should be added for the internal function.
if (!CtxKey->Context.empty()) {
// Get or create function profile for the range
- FunctionProfile = &getFunctionProfileForContext(CtxKey->Context,
- CtxKey->WasLeafInlined);
+ ContextNode =
+ getOrCreateContextNode(CtxKey->Context, CtxKey->WasLeafInlined);
// Fill in function body samples
- populateBodySamplesForFunction(*FunctionProfile, CI.second.RangeCounter);
+ populateBodySamplesForFunction(*ContextNode->getFunctionSamples(),
+ CI.second.RangeCounter);
}
// Fill in boundary sample counts as well as call site samples for calls
- populateBoundarySamplesForFunction(CtxKey->Context, FunctionProfile,
- CI.second.BranchCounter);
+ populateBoundarySamplesForFunction(ContextNode, CI.second.BranchCounter);
}
// Fill in call site value sample for inlined calls and also use context to
// infer missing samples. Since we don't have call count for inlined
// functions, we estimate it from inlinee's profile using the entry of the
// body sample.
- populateInferredFunctionSamples();
+ populateInferredFunctionSamples(getRootContext());
updateFunctionSamples();
}
@@ -834,8 +857,7 @@ void CSProfileGenerator::populateBodySamplesForFunction(
}
void CSProfileGenerator::populateBoundarySamplesForFunction(
- SampleContextFrames ContextId, FunctionSamples *CallerProfile,
- const BranchSample &BranchCounters) {
+ ContextTrieNode *Node, const BranchSample &BranchCounters) {
for (const auto &Entry : BranchCounters) {
uint64_t SourceOffset = Entry.first.first;
@@ -847,88 +869,105 @@ void CSProfileGenerator::populateBoundarySamplesForFunction(
if (CalleeName.size() == 0)
continue;
- SampleContextFrameVector CalleeCtx;
- if (CallerProfile) {
- assert(!ContextId.empty() &&
- "CallerProfile is null only if ContextId is empty");
+ ContextTrieNode *CallerNode = Node;
+ LineLocation CalleeCallSite(0, 0);
+ if (CallerNode != &getRootContext()) {
// Record called target sample and its count
auto LeafLoc = Binary->getInlineLeafFrameLoc(SourceOffset);
if (LeafLoc) {
- CallerProfile->addCalledTargetSamples(
+ CallerNode->getFunctionSamples()->addCalledTargetSamples(
LeafLoc->Location.LineOffset,
getBaseDiscriminator(LeafLoc->Location.Discriminator), CalleeName,
Count);
-
// Record head sample for called target(callee)
- CalleeCtx.append(ContextId.begin(), ContextId.end());
- assert(CalleeCtx.back().FuncName == LeafLoc->FuncName &&
- "Leaf function name doesn't match");
- CalleeCtx.back() = *LeafLoc;
+ CalleeCallSite = LeafLoc->Location;
}
}
- CalleeCtx.emplace_back(CalleeName, LineLocation(0, 0));
- FunctionSamples &CalleeProfile = getFunctionProfileForContext(CalleeCtx);
- CalleeProfile.addHeadSamples(Count);
+
+ ContextTrieNode *CalleeNode =
+ CallerNode->getOrCreateChildContext(CalleeCallSite, CalleeName);
+ FunctionSamples *CalleeProfile = getOrCreateFunctionSamples(CalleeNode);
+ CalleeProfile->addHeadSamples(Count);
}
}
-static SampleContextFrame
-getCallerContext(SampleContextFrames CalleeContext,
- SampleContextFrameVector &CallerContext) {
- assert(CalleeContext.size() > 1 && "Unexpected empty context");
- CalleeContext = CalleeContext.drop_back();
- CallerContext.assign(CalleeContext.begin(), CalleeContext.end());
- SampleContextFrame CallerFrame = CallerContext.back();
- CallerContext.back().Location = LineLocation(0, 0);
- return CallerFrame;
-}
+void CSProfileGenerator::populateInferredFunctionSamples(
+ ContextTrieNode &Node) {
+ // There is no call jmp sample between the inliner and inlinee, we need to use
+ // the inlinee's context to infer inliner's context, i.e. parent(inliner)'s
+ // sample depends on child(inlinee)'s sample, so traverse the tree in
+ // post-order.
+ for (auto &It : Node.getAllChildContext())
+ populateInferredFunctionSamples(It.second);
+
+ FunctionSamples *CalleeProfile = Node.getFunctionSamples();
+ if (!CalleeProfile)
+ return;
+ // If we already have head sample counts, we must have value profile
+ // for call sites added already. Skip to avoid double counting.
+ if (CalleeProfile->getHeadSamples())
+ return;
+ ContextTrieNode *CallerNode = Node.getParentContext();
+ // If we don't have context, nothing to do for caller's call site.
+ // This could happen for entry point function.
+ if (CallerNode == &getRootContext())
+ return;
-void CSProfileGenerator::populateInferredFunctionSamples() {
- for (const auto &Item : ProfileMap) {
- const auto &CalleeContext = Item.first;
- const FunctionSamples &CalleeProfile = Item.second;
+ LineLocation CallerLeafFrameLoc = Node.getCallSiteLoc();
+ FunctionSamples &CallerProfile = *getOrCreateFunctionSamples(CallerNode);
+ // Since we don't have call count for inlined functions, we
+ // estimate it from inlinee's profile using entry body sample.
+ uint64_t EstimatedCallCount = CalleeProfile->getEntrySamples();
+ // If we don't have samples with location, use 1 to indicate live.
+ if (!EstimatedCallCount && !CalleeProfile->getBodySamples().size())
+ EstimatedCallCount = 1;
+ CallerProfile.addCalledTargetSamples(CallerLeafFrameLoc.LineOffset,
+ CallerLeafFrameLoc.Discriminator,
+ Node.getFuncName(), EstimatedCallCount);
+ CallerProfile.addBodySamples(CallerLeafFrameLoc.LineOffset,
+ CallerLeafFrameLoc.Discriminator,
+ EstimatedCallCount);
+ CallerProfile.addTotalSamples(EstimatedCallCount);
+}
- // If we already have head sample counts, we must have value profile
- // for call sites added already. Skip to avoid double counting.
- if (CalleeProfile.getHeadSamples())
- continue;
- // If we don't have context, nothing to do for caller's call site.
- // This could happen for entry point function.
- if (CalleeContext.isBaseContext())
- continue;
+void CSProfileGenerator::convertToProfileMap(
+ ContextTrieNode &Node, SampleContextFrameVector &Context) {
+ FunctionSamples *FProfile = Node.getFunctionSamples();
+ if (FProfile) {
+ Context.emplace_back(Node.getFuncName(), LineLocation(0, 0));
+ // Save the new context for future references.
+ SampleContextFrames NewContext = *Contexts.insert(Context).first;
+ auto Ret = ProfileMap.emplace(NewContext, std::move(*FProfile));
+ FunctionSamples &NewProfile = Ret.first->second;
+ NewProfile.getContext().setContext(NewContext);
+ Context.pop_back();
+ }
- // Infer Caller's frame loc and context ID through string splitting
- SampleContextFrameVector CallerContextId;
- SampleContextFrame &&CallerLeafFrameLoc =
- getCallerContext(CalleeContext.getContextFrames(), CallerContextId);
- SampleContextFrames CallerContext(CallerContextId);
-
- // It's possible that we haven't seen any sample directly in the caller,
- // in which case CallerProfile will not exist. But we can't modify
- // ProfileMap while iterating it.
- // TODO: created function profile for those callers too
- if (ProfileMap.find(CallerContext) == ProfileMap.end())
- continue;
- FunctionSamples &CallerProfile = ProfileMap[CallerContext];
-
- // Since we don't have call count for inlined functions, we
- // estimate it from inlinee's profile using entry body sample.
- uint64_t EstimatedCallCount = CalleeProfile.getEntrySamples();
- // If we don't have samples with location, use 1 to indicate live.
- if (!EstimatedCallCount && !CalleeProfile.getBodySamples().size())
- EstimatedCallCount = 1;
- CallerProfile.addCalledTargetSamples(
- CallerLeafFrameLoc.Location.LineOffset,
- CallerLeafFrameLoc.Location.Discriminator,
- CalleeProfile.getContext().getName(), EstimatedCallCount);
- CallerProfile.addBodySamples(CallerLeafFrameLoc.Location.LineOffset,
- CallerLeafFrameLoc.Location.Discriminator,
- EstimatedCallCount);
- CallerProfile.addTotalSamples(EstimatedCallCount);
+ for (auto &It : Node.getAllChildContext()) {
+ ContextTrieNode &ChildNode = It.second;
+ Context.emplace_back(Node.getFuncName(), ChildNode.getCallSiteLoc());
+ convertToProfileMap(ChildNode, Context);
+ Context.pop_back();
}
}
+void CSProfileGenerator::convertToProfileMap() {
+ assert(ProfileMap.empty() &&
+ "ProfileMap should be empty before converting from the trie");
+ assert(IsProfileValidOnTrie &&
+ "Do not convert the trie twice, it's already destroyed");
+
+ SampleContextFrameVector Context;
+ for (auto &It : getRootContext().getAllChildContext())
+ convertToProfileMap(It.second, Context);
+
+ IsProfileValidOnTrie = false;
+}
+
void CSProfileGenerator::postProcessProfiles() {
+ if (SampleCounters)
+ convertToProfileMap();
+
// Compute hot/cold threshold based on profile. This will be used for cold
// context profile merging/trimming.
computeSummaryAndThreshold();
@@ -1069,8 +1108,10 @@ void CSProfileGenerator::populateBodySamplesWithProbes(
// doesn't belong to current context, filter them out.
if (!Probe->isBlock() || Count == 0)
continue;
- FunctionSamples &FunctionProfile =
- getFunctionProfileForLeafProbe(ContextStack, Probe);
+
+ ContextTrieNode *ContextNode =
+ getContextNodeForLeafProbe(ContextStack, Probe);
+ FunctionSamples &FunctionProfile = *ContextNode->getFunctionSamples();
// Record the current frame and FunctionProfile whenever samples are
// collected for non-danglie probes. This is for reporting all of the
// zero count probes of the frame later.
@@ -1081,25 +1122,21 @@ void CSProfileGenerator::populateBodySamplesWithProbes(
FunctionProfile.addHeadSamples(Count);
// Look up for the caller's function profile
const auto *InlinerDesc = Binary->getInlinerDescForProbe(Probe);
- SampleContextFrames CalleeContextId =
- FunctionProfile.getContext().getContextFrames();
- if (InlinerDesc != nullptr && CalleeContextId.size() > 1) {
+ ContextTrieNode *CallerNode = ContextNode->getParentContext();
+ if (InlinerDesc != nullptr && CallerNode != &getRootContext()) {
// Since the context id will be compressed, we have to use callee's
// context id to infer caller's context id to ensure they share the
// same context prefix.
- SampleContextFrameVector CallerContextId;
- SampleContextFrame &&CallerLeafFrameLoc =
- getCallerContext(CalleeContextId, CallerContextId);
- uint64_t CallerIndex = CallerLeafFrameLoc.Location.LineOffset;
+ uint64_t CallerIndex = ContextNode->getCallSiteLoc().LineOffset;
assert(CallerIndex &&
"Inferred caller's location index shouldn't be zero!");
FunctionSamples &CallerProfile =
- getFunctionProfileForContext(CallerContextId);
+ *getOrCreateFunctionSamples(CallerNode);
CallerProfile.setFunctionHash(InlinerDesc->FuncHash);
CallerProfile.addBodySamples(CallerIndex, 0, Count);
CallerProfile.addTotalSamples(Count);
- CallerProfile.addCalledTargetSamples(
- CallerIndex, 0, FunctionProfile.getContext().getName(), Count);
+ CallerProfile.addCalledTargetSamples(CallerIndex, 0,
+ ContextNode->getFuncName(), Count);
}
}
}
@@ -1139,7 +1176,7 @@ void CSProfileGenerator::populateBoundarySamplesWithProbes(
}
}
-FunctionSamples &CSProfileGenerator::getFunctionProfileForLeafProbe(
+ContextTrieNode *CSProfileGenerator::getContextNodeForLeafProbe(
SampleContextFrames ContextStack, const MCDecodedPseudoProbe *LeafProbe) {
// Explicitly copy the context for appending the leaf context
@@ -1159,10 +1196,16 @@ FunctionSamples &CSProfileGenerator::getFunctionProfileForLeafProbe(
const auto *FuncDesc = Binary->getFuncDescForGUID(LeafProbe->getGuid());
bool WasLeafInlined = LeafProbe->getInlineTreeNode()->hasInlineSite();
- FunctionSamples &FunctionProile =
- getFunctionProfileForContext(NewContextStack, WasLeafInlined);
- FunctionProile.setFunctionHash(FuncDesc->FuncHash);
- return FunctionProile;
+ ContextTrieNode *ContextNode =
+ getOrCreateContextNode(NewContextStack, WasLeafInlined);
+ ContextNode->getFunctionSamples()->setFunctionHash(FuncDesc->FuncHash);
+ return ContextNode;
+}
+
+FunctionSamples &CSProfileGenerator::getFunctionProfileForLeafProbe(
+ SampleContextFrames ContextStack, const MCDecodedPseudoProbe *LeafProbe) {
+ return *getContextNodeForLeafProbe(ContextStack, LeafProbe)
+ ->getFunctionSamples();
}
} // end namespace sampleprof
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
index a6a9a10baadcb..f26c3d0d82097 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.h
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -294,10 +294,15 @@ class CSProfileGenerator : public ProfileGeneratorBase {
private:
void generateLineNumBasedProfile();
- // Lookup or create FunctionSamples for the context
- FunctionSamples &
- getFunctionProfileForContext(const SampleContextFrameVector &Context,
- bool WasLeafInlined = false);
+
+ FunctionSamples *getOrCreateFunctionSamples(ContextTrieNode *ContextNode,
+ bool WasLeafInlined = false);
+
+ // Lookup or create ContextTrieNode for the context, FunctionSamples is
+ // created inside this function.
+ ContextTrieNode *getOrCreateContextNode(const SampleContextFrames Context,
+ bool WasLeafInlined = false);
+
// For profiled only functions, on-demand compute their inline context
// function byte size which is used by the pre-inliner.
void computeSizeForProfiledFunctions();
@@ -307,10 +312,13 @@ class CSProfileGenerator : public ProfileGeneratorBase {
void populateBodySamplesForFunction(FunctionSamples &FunctionProfile,
const RangeSample &RangeCounters);
- void populateBoundarySamplesForFunction(SampleContextFrames ContextId,
- FunctionSamples *CallerProfile,
+
+ void populateBoundarySamplesForFunction(ContextTrieNode *CallerNode,
const BranchSample &BranchCounters);
- void populateInferredFunctionSamples();
+
+ void populateInferredFunctionSamples(ContextTrieNode &Node);
+
+ void updateFunctionSamples();
void generateProbeBasedProfile();
@@ -320,14 +328,33 @@ class CSProfileGenerator : public ProfileGeneratorBase {
// Fill in boundary samples for a call probe
void populateBoundarySamplesWithProbes(const BranchSample &BranchCounter,
SampleContextFrames ContextStack);
+
+ ContextTrieNode *
+ getContextNodeForLeafProbe(SampleContextFrames ContextStack,
+ const MCDecodedPseudoProbe *LeafProbe);
+
// Helper function to get FunctionSamples for the leaf probe
FunctionSamples &
getFunctionProfileForLeafProbe(SampleContextFrames ContextStack,
const MCDecodedPseudoProbe *LeafProbe);
+ void convertToProfileMap(ContextTrieNode &Node,
+ SampleContextFrameVector &Context);
+
+ void convertToProfileMap();
+
+ ContextTrieNode &getRootContext() { return ContextTracker.getRootContext(); };
+
+ // The container for holding the FunctionSamples used by context trie.
+ std::list<FunctionSamples> FSamplesList;
+
// Underlying context table serves for sample profile writer.
std::unordered_set<SampleContextFrameVector, SampleContextFrameHash> Contexts;
+ SampleContextTracker ContextTracker;
+
+ bool IsProfileValidOnTrie = true;
+
public:
// Deduplicate adjacent repeated context sequences up to a given sequence
// length. -1 means no size limit.
More information about the llvm-commits
mailing list