[llvm] [SampleFDO] Stale profile call-graph matching (PR #92151)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 4 00:49:50 PDT 2024
================
@@ -590,14 +618,308 @@ void SampleProfileMatcher::computeAndReportProfileStaleness() {
}
}
-void SampleProfileMatcher::runOnModule() {
- ProfileConverter::flattenProfile(Reader.getProfiles(), FlattenedProfiles,
- FunctionSamples::ProfileIsCS);
+void SampleProfileMatcher::findNewIRFunctions(
+ StringMap<Function *> &NewIRFunctions) {
+ // TODO: Support MD5 profile.
+ if (FunctionSamples::UseMD5)
+ return;
+ StringSet<> NamesInProfile;
+ if (auto NameTable = Reader.getNameTable()) {
+ for (auto Name : *NameTable)
+ NamesInProfile.insert(Name.stringRef());
+ }
+
for (auto &F : M) {
- if (skipProfileForFunction(F))
+ // Skip declarations, as even if the function can be matched, we have
+ // nothing to do with it.
+ if (F.isDeclaration())
continue;
- runOnFunction(F);
+
+ StringRef CanonFName = FunctionSamples::getCanonicalFnName(F.getName());
+ const auto *FS = getFlattenedSamplesFor(F);
+ if (FS)
+ continue;
+
+ // For extended binary, functions are fully inlined may not be loaded in the
+ // top-level profile, so check the NameTable which has the all symbol names
+ // in profile.
+ if (NamesInProfile.count(CanonFName))
+ continue;
+
+ // For extended binary, non-profiled function symbols are in the profile
+ // symbol list table.
+ if (PSL && PSL->contains(CanonFName))
+ continue;
+
+ LLVM_DEBUG(dbgs() << "Function " << CanonFName
+ << " is not in profile or symbol list table.\n");
+ NewIRFunctions[CanonFName] = &F;
+ }
+}
+
+std::vector<Function *> *SampleProfileMatcher::findNewIRCallees(
+ Function &Func, const StringMap<Function *> &NewIRFunctions) {
+ auto R = FuncToNewCalleesMap.try_emplace(&Func, std::vector<Function *>());
+ std::vector<Function *> &IRCalleesToMatch = R.first->second;
+ // Skip the lookup if it's in the cache.
+ if (!R.second)
+ return &IRCalleesToMatch;
+
+ for (auto &BB : Func) {
+ for (auto &I : BB) {
+ const auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB || isa<IntrinsicInst>(&I))
+ continue;
+ Function *Callee = CB->getCalledFunction();
+ if (!Callee || Callee->isDeclaration())
+ continue;
+ StringRef CalleeName =
+ FunctionSamples::getCanonicalFnName(Callee->getName());
+ if (NewIRFunctions.count(CalleeName))
+ IRCalleesToMatch.push_back(Callee);
+ }
+ }
+ return &IRCalleesToMatch;
+}
+
+Function *SampleProfileMatcher::findFuncByProfileName(
+ const FunctionId &ProfileName) const {
+ auto F = SymbolMap->find(ProfileName);
+ if (F != SymbolMap->end())
+ return F->second;
+
+ // Find in new matched function map.
+ auto NewF = ProfileNameToFuncMap.find(ProfileName);
+ if (NewF != ProfileNameToFuncMap.end())
+ return NewF->second;
+ return nullptr;
+}
+
+bool SampleProfileMatcher::functionMatchesProfileHelper(
+ const Function &IRFunc, const FunctionId &ProfFunc) {
+ // The value is in the range [0, 1]. The bigger the value is, the more similar
+ // two sequences are.
+ float Similarity = 0.0;
+
+ const auto *FSFlattened = getFlattenedSamplesFor(ProfFunc);
+ assert(FSFlattened && "Flattened profile sample is null");
+ // Similarity check may not be reliable if the function is tiny, we use the
+ // number of basic block as a proxy for the function complexity and skip the
+ // matching if it's too small.
+ if (IRFunc.size() < MinFuncCountForCGMatching ||
+ FSFlattened->getBodySamples().size() < MinFuncCountForCGMatching)
+ return false;
+
+ // For probe-based function, we first trust the checksum info. If the checksum
+ // doesn't match, we continue checking for similarity.
+ if (FunctionSamples::ProfileIsProbeBased) {
+ const auto *FuncDesc = ProbeManager->getDesc(IRFunc);
+ if (FuncDesc &&
+ !ProbeManager->profileIsHashMismatched(*FuncDesc, *FSFlattened)) {
+ LLVM_DEBUG(dbgs() << "The checksums for " << IRFunc.getName()
+ << "(IR) and " << ProfFunc << "(Profile) match.\n");
+
+ return true;
+ }
+ }
+
+ AnchorMap IRAnchors;
+ findIRAnchors(IRFunc, IRAnchors);
+ AnchorMap ProfileAnchors;
+ findProfileAnchors(*FSFlattened, ProfileAnchors);
+
+ AnchorList FilteredIRAnchorsList;
+ AnchorList FilteredProfileAnchorList;
+ getFilteredAnchorList(IRAnchors, ProfileAnchors, FilteredIRAnchorsList,
+ FilteredProfileAnchorList);
+
+ // Similarly skip the matching if the num of anchors is not enough.
+ if (FilteredIRAnchorsList.size() < MinCallCountForCGMatching ||
+ FilteredProfileAnchorList.size() < MinCallCountForCGMatching)
+ return false;
+
+ // Use the diff algorithm to find the LCS between IR and profile.
+ LocToLocMap MatchedAnchors =
+ longestCommonSequence(FilteredIRAnchorsList, FilteredProfileAnchorList);
+
+ Similarity =
+ static_cast<float>(MatchedAnchors.size()) * 2 /
+ (FilteredIRAnchorsList.size() + FilteredProfileAnchorList.size());
+
+ LLVM_DEBUG(dbgs() << "The similarity between " << IRFunc.getName()
+ << "(IR) and " << ProfFunc << "(profile) is "
+ << format("%.2f", Similarity) << "\n");
+ assert((Similarity >= 0 && Similarity <= 1.0) &&
+ "Similarity value should be in [0, 1]");
+ return Similarity * 100 > FuncProfileSimilarityThreshold;
+}
+
+bool SampleProfileMatcher::functionMatchesProfile(Function &IRFunc,
+ const FunctionId &ProfFunc) {
+ auto R = FunctionProfileNameMap.find({&IRFunc, ProfFunc});
+ if (R != FunctionProfileNameMap.end())
+ return R->second;
+
+ bool Matched = functionMatchesProfileHelper(IRFunc, ProfFunc);
+ FunctionProfileNameMap[{&IRFunc, ProfFunc}] = Matched;
+ ProfileNameToFuncMap[ProfFunc] = &IRFunc;
+ return Matched;
+}
+
+void SampleProfileMatcher::matchCalleeProfile(
+ const FunctionId &Caller, const FunctionId &ProfileCalleeName,
+ const std::vector<Function *> *IRCalleesToMatch,
+ std::vector<std::pair<FunctionId, FunctionId>> &MatchResult) {
+ if (!IRCalleesToMatch)
+ return;
+ // Check whether this is an existing function matching the profile, we only
+ // run the matching when the callee profile is unused.
+ auto F = SymbolMap->find(ProfileCalleeName);
+ if (F != SymbolMap->end())
+ return;
+
+ for (auto *IRFuncCandidate : *IRCalleesToMatch)
+ if (functionMatchesProfile(*IRFuncCandidate, ProfileCalleeName)) {
+ FunctionId IRCalleeName(IRFuncCandidate->getName());
+ assert(IRCalleeName != ProfileCalleeName &&
+ "New callee symbol is not a new function");
+ LLVM_DEBUG(dbgs() << "In function " << Caller
+ << ", changing profile name from " << ProfileCalleeName
+ << " to " << IRCalleeName << "\n");
+ MatchResult.emplace_back(IRCalleeName, ProfileCalleeName);
+ return;
+ }
+}
+
+// Traverse the profiled call-graph recursively to run the matching.
+void SampleProfileMatcher::matchProfileForNewFunctions(
+ const StringMap<Function *> &NewIRFunctions, FunctionSamples &FuncProfile) {
+ // Find the new candidate IR callees in the current caller scope.
+ std::vector<Function *> *IRCalleesToMatch = nullptr;
+ if (auto *IRCaller = findFuncByProfileName(FuncProfile.getFunction())) {
+ // No callees for external function, skip the call graph matching.
+ if (IRCaller->isDeclaration())
+ return;
+ IRCalleesToMatch = findNewIRCallees(*IRCaller, NewIRFunctions);
+ }
+ // Don't return here when IRCalleesToMatch is nullptr or empty, this is
+ // because even if there is no matching in the current scope, there could be
+ // matching in deeper callee scope/edge, so we need to keep traversing the
+ // call-graph. For IRCalleesToMatch is nullptr or empty case, later the
+ // matching function(matchCalleeProfile) will handle this to make it non-op.
+
+ // Match non-inline callees.
+ for (auto &BS : const_cast<BodySampleMap &>(FuncProfile.getBodySamples())) {
+ // New function to old function pairs used to update the CallTargetMap.
+ std::vector<std::pair<FunctionId, FunctionId>> MatchResult;
+ SampleRecord::CallTargetMap &CTM =
+ const_cast<SampleRecord::CallTargetMap &>(BS.second.getCallTargets());
+ for (const auto &TS : CTM)
+ matchCalleeProfile(FuncProfile.getFunction(), TS.first, IRCalleesToMatch,
+ MatchResult);
+
+ // Update the CallTargetMap.
+ for (const auto &P : MatchResult) {
+ CTM[P.first] = CTM[P.second];
+ CTM.erase(P.second);
+ }
+ }
+
+ // Match inline callees.
+ for (auto &CM :
+ const_cast<CallsiteSampleMap &>(FuncProfile.getCallsiteSamples())) {
+ auto &CalleeMap = CM.second;
+ // New function to old function pairs used to update the CallsiteSampleMap.
+ std::vector<std::pair<FunctionId, FunctionId>> MatchResult;
+ for (auto &CS : CalleeMap) {
+ FunctionSamples &CalleeProfile = CS.second;
+ matchCalleeProfile(FuncProfile.getFunction(), CalleeProfile.getFunction(),
+ IRCalleesToMatch, MatchResult);
+
+ // Traverse all the inlined callee profiles.
+ matchProfileForNewFunctions(NewIRFunctions, CalleeProfile);
+ }
+
+ // Update the CalleeMap using the new name and remove the old entry.
+ for (auto &P : MatchResult) {
+ assert(P.first != P.second &&
+ "Renamed function name should be different from the old map key");
+ FunctionSamples &FS = CalleeMap[P.second];
+ FS.setFunction(P.first);
+ CalleeMap[P.first] = FS;
+ CalleeMap.erase(P.second);
+ }
}
+}
+
+// Match the unused profile with new IR functions on the profiled call-graph.
+// The high-level steps for the algorithm:
+// 1) Find all the new functions that show only in the IR, use them as the
+// matching candidates to compute new callees.
+//
+// 2) Traverse all the nodes in the profiled call-graph.
+// For each function caller scope:
+// a) Find a set of callees in the IR that doesn't exist in the profile. See
+// findNewIRCallees.
+// b) Find a set of callees in the profile that doesn't exist
+// in the IR. See matchCalleeProfile.
+// c) Match the callee pairs between a and b. Compute a similarity ratio
+// between the pair, it's considered match if the similarity is above a given
+// threshold. See MatchProfileForNewFunctions.
+// d) Update the profile with the matched name in-place.
+void SampleProfileMatcher::runCallGraphMatching() {
+ if (!SalvageUnusedProfile)
+ return;
+ assert(SymbolMap && "SymbolMap is null");
+ assert(FunctionProfileNameMap.empty() &&
+ "FunctionProfileNameMap is not empty before the call graph matching");
+
+ StringMap<Function *> NewIRFunctions;
+ findNewIRFunctions(NewIRFunctions);
+ if (NewIRFunctions.empty())
+ return;
+
+ // Sort the profiles to make the matching order deterministic.
+ std::vector<NameFunctionSamples> SortedProfiles;
+ ::llvm::sortFuncProfiles(Reader.getProfiles(), SortedProfiles);
+ for (auto &P : SortedProfiles)
+ matchProfileForNewFunctions(NewIRFunctions,
+ *const_cast<FunctionSamples *>(P.second));
+
+ clearCacheData();
----------------
WenleiHe wrote:
nit: inline it here and comment explaining this is for freeing memory (?)
https://github.com/llvm/llvm-project/pull/92151
More information about the llvm-commits
mailing list