[llvm] 96cb97c - [ThinLTO] Update combined index for SamplePGO indirect calls to locals
Teresa Johnson via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 24 12:29:59 PDT 2021
Author: Teresa Johnson
Date: 2021-09-24T12:29:49-07:00
New Revision: 96cb97c4533a0a02c2d62ffb1121cd275aa43dd5
URL: https://github.com/llvm/llvm-project/commit/96cb97c4533a0a02c2d62ffb1121cd275aa43dd5
DIFF: https://github.com/llvm/llvm-project/commit/96cb97c4533a0a02c2d62ffb1121cd275aa43dd5.diff
LOG: [ThinLTO] Update combined index for SamplePGO indirect calls to locals
In ThinLTO for locals we normally compute the GUID from the name after
prepending the source path to get a unique global id. SamplePGO indirect
call profiles contain the target GUID without this uniquification,
however (unless compiling with -funique-internal-linkage-names).
In order to correctly handle the call edges added to the combined index
for these indirect calls, during importing and bitcode writing we
consult a map of original to full GUID to identify the actual callee.
However, for a large application this was consuming a lot of compile
time as we need to do this repeatedly (especially during importing where
we may traverse call edges multiple times).
To fix this implement a suggestion in one of the FIXME comments, and
actually modify the call edges during a single traversal after the index
is built to perform the fixups once. I combined this fixup with the dead
code analysis performed on the index in order to avoid adding an
additional walk of the index. The dead code analysis is the first
analysis performed on the index.
This reduced the time required for a large thin link with SamplePGO by
about 20%.
No new test added, but I confirmed that there are existing tests that
will fail when no fixup is performed.
Differential Revision: https://reviews.llvm.org/D110374
Added:
Modified:
llvm/include/llvm/IR/ModuleSummaryIndex.h
llvm/include/llvm/Transforms/IPO/FunctionImport.h
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/IR/ModuleSummaryIndex.cpp
llvm/lib/Transforms/IPO/FunctionImport.cpp
llvm/tools/llvm-lto/llvm-lto.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index 4b84f6b0408de..2f2afab6285f9 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -700,6 +700,8 @@ class FunctionSummary : public GlobalValueSummary {
/// Return the list of <CalleeValueInfo, CalleeInfo> pairs.
ArrayRef<EdgeTy> calls() const { return CallGraphEdgeList; }
+ std::vector<EdgeTy> &mutableCalls() { return CallGraphEdgeList; }
+
void addCall(EdgeTy E) { CallGraphEdgeList.push_back(E); }
/// Returns the list of type identifiers used by this function in
diff --git a/llvm/include/llvm/Transforms/IPO/FunctionImport.h b/llvm/include/llvm/Transforms/IPO/FunctionImport.h
index aad938d48570a..fa2a5135b9b6d 100644
--- a/llvm/include/llvm/Transforms/IPO/FunctionImport.h
+++ b/llvm/include/llvm/Transforms/IPO/FunctionImport.h
@@ -167,16 +167,24 @@ void ComputeCrossModuleImportForModuleFromIndex(
FunctionImporter::ImportMapTy &ImportList);
/// PrevailingType enum used as a return type of callback passed
-/// to computeDeadSymbols. Yes and No values used when status explicitly
-/// set by symbols resolution, otherwise status is Unknown.
+/// to computeDeadSymbolsAndUpdateIndirectCalls. Yes and No values used when
+/// status explicitly set by symbols resolution, otherwise status is Unknown.
enum class PrevailingType { Yes, No, Unknown };
+/// Update call edges for indirect calls to local functions added from
+/// SamplePGO when needed. Normally this is done during
+/// computeDeadSymbolsAndUpdateIndirectCalls, but can be called standalone
+/// when that is not called (e.g. during testing).
+void updateIndirectCalls(ModuleSummaryIndex &Index);
+
/// Compute all the symbols that are "dead": i.e these that can't be reached
/// in the graph from any of the given symbols listed in
/// \p GUIDPreservedSymbols. Non-prevailing symbols are symbols without a
/// prevailing copy anywhere in IR and are normally dead, \p isPrevailing
/// predicate returns status of symbol.
-void computeDeadSymbols(
+/// Also update call edges for indirect calls to local functions added from
+/// SamplePGO when needed.
+void computeDeadSymbolsAndUpdateIndirectCalls(
ModuleSummaryIndex &Index,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing);
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 402840eab6a28..9ddc4a22334ed 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -4208,33 +4208,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
}
auto GetValueId = [&](const ValueInfo &VI) -> Optional<unsigned> {
- GlobalValue::GUID GUID = VI.getGUID();
- Optional<unsigned> CallValueId = getValueId(GUID);
- if (CallValueId)
- return CallValueId;
- // For SamplePGO, the indirect call targets for local functions will
- // have its original name annotated in profile. We try to find the
- // corresponding PGOFuncName as the GUID.
- GUID = Index.getGUIDFromOriginalID(GUID);
- if (!GUID)
- return None;
- CallValueId = getValueId(GUID);
- if (!CallValueId)
- return None;
- // The mapping from OriginalId to GUID may return a GUID
- // that corresponds to a static variable. Filter it out here.
- // This can happen when
- // 1) There is a call to a library function which does not have
- // a CallValidId;
- // 2) There is a static variable with the OriginalGUID identical
- // to the GUID of the library function in 1);
- // When this happens, the logic for SamplePGO kicks in and
- // the static variable in 2) will be found, which needs to be
- // filtered out.
- auto *GVSum = Index.getGlobalValueSummary(GUID, false);
- if (GVSum && GVSum->getSummaryKind() == GlobalValueSummary::GlobalVarKind)
- return None;
- return CallValueId;
+ return getValueId(VI.getGUID());
};
auto *FS = cast<FunctionSummary>(S);
diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index f4ac6caf4f93a..b003ebc401f84 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -251,12 +251,13 @@ void ModuleSummaryIndex::propagateAttributes(
bool IsDSOLocal = true;
for (auto &S : P.second.SummaryList) {
if (!isGlobalValueLive(S.get())) {
- // computeDeadSymbols should have marked all copies live. Note that
- // it is possible that there is a GUID collision between internal
- // symbols with the same name in
diff erent files of the same name but
- // not enough distinguishing path. Because computeDeadSymbols should
- // conservatively mark all copies live we can assert here that all are
- // dead if any copy is dead.
+ // computeDeadSymbolsAndUpdateIndirectCalls should have marked all
+ // copies live. Note that it is possible that there is a GUID collision
+ // between internal symbols with the same name in
diff erent files of the
+ // same name but not enough distinguishing path. Because
+ // computeDeadSymbolsAndUpdateIndirectCalls should conservatively mark
+ // all copies live we can assert here that all are dead if any copy is
+ // dead.
assert(llvm::none_of(
P.second.SummaryList,
[&](const std::unique_ptr<GlobalValueSummary> &Summary) {
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index 4535b75e2c482..b4b5f733a7c66 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -188,23 +188,6 @@ selectCallee(const ModuleSummaryIndex &Index,
return false;
}
- // For SamplePGO, in computeImportForFunction the OriginalId
- // may have been used to locate the callee summary list (See
- // comment there).
- // The mapping from OriginalId to GUID may return a GUID
- // that corresponds to a static variable. Filter it out here.
- // This can happen when
- // 1) There is a call to a library function which is not defined
- // in the index.
- // 2) There is a static variable with the OriginalGUID identical
- // to the GUID of the library function in 1);
- // When this happens, the logic for SamplePGO kicks in and
- // the static variable in 2) will be found, which needs to be
- // filtered out.
- if (GVSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind) {
- Reason = FunctionImporter::ImportFailureReason::GlobalVar;
- return false;
- }
if (GlobalValue::isInterposableLinkage(GVSummary->linkage())) {
Reason = FunctionImporter::ImportFailureReason::InterposableLinkage;
// There is no point in importing these, we can't inline them
@@ -265,21 +248,6 @@ using EdgeInfo =
} // anonymous namespace
-static ValueInfo
-updateValueInfoForIndirectCalls(const ModuleSummaryIndex &Index, ValueInfo VI) {
- if (!VI.getSummaryList().empty())
- return VI;
- // For SamplePGO, the indirect call targets for local functions will
- // have its original name annotated in profile. We try to find the
- // corresponding PGOFuncName as the GUID.
- // FIXME: Consider updating the edges in the graph after building
- // it, rather than needing to perform this mapping on each walk.
- auto GUID = Index.getGUIDFromOriginalID(VI.getGUID());
- if (GUID == 0)
- return ValueInfo();
- return Index.getValueInfo(GUID);
-}
-
static bool shouldImportGlobal(const ValueInfo &VI,
const GVSummaryMapTy &DefinedGVSummaries) {
const auto &GVS = DefinedGVSummaries.find(VI.getGUID());
@@ -401,10 +369,6 @@ static void computeImportForFunction(
continue;
}
- VI = updateValueInfoForIndirectCalls(Index, VI);
- if (!VI)
- continue;
-
if (DefinedGVSummaries.count(VI.getGUID())) {
// FIXME: Consider not skipping import if the module contains
// a non-prevailing def with interposable linkage. The prevailing copy
@@ -840,16 +804,61 @@ void llvm::ComputeCrossModuleImportForModuleFromIndex(
#endif
}
-void llvm::computeDeadSymbols(
+// For SamplePGO, the indirect call targets for local functions will
+// have its original name annotated in profile. We try to find the
+// corresponding PGOFuncName as the GUID, and fix up the edges
+// accordingly.
+void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index,
+ FunctionSummary *FS) {
+ for (auto &EI : FS->mutableCalls()) {
+ if (!EI.first.getSummaryList().empty())
+ continue;
+ auto GUID = Index.getGUIDFromOriginalID(EI.first.getGUID());
+ if (GUID == 0)
+ continue;
+ // Update the edge to point directly to the correct GUID.
+ auto VI = Index.getValueInfo(GUID);
+ if (llvm::any_of(
+ VI.getSummaryList(),
+ [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
+ // The mapping from OriginalId to GUID may return a GUID
+ // that corresponds to a static variable. Filter it out here.
+ // This can happen when
+ // 1) There is a call to a library function which is not defined
+ // in the index.
+ // 2) There is a static variable with the OriginalGUID identical
+ // to the GUID of the library function in 1);
+ // When this happens the static variable in 2) will be found,
+ // which needs to be filtered out.
+ return SummaryPtr->getSummaryKind() ==
+ GlobalValueSummary::GlobalVarKind;
+ }))
+ continue;
+ EI.first = VI;
+ }
+}
+
+void llvm::updateIndirectCalls(ModuleSummaryIndex &Index) {
+ for (const auto &Entry : Index) {
+ for (auto &S : Entry.second.SummaryList) {
+ if (auto *FS = dyn_cast<FunctionSummary>(S.get()))
+ updateValueInfoForIndirectCalls(Index, FS);
+ }
+ }
+}
+
+void llvm::computeDeadSymbolsAndUpdateIndirectCalls(
ModuleSummaryIndex &Index,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
assert(!Index.withGlobalValueDeadStripping());
- if (!ComputeDead)
- return;
- if (GUIDPreservedSymbols.empty())
- // Don't do anything when nothing is live, this is friendly with tests.
+ if (!ComputeDead ||
+ // Don't do anything when nothing is live, this is friendly with tests.
+ GUIDPreservedSymbols.empty()) {
+ // Still need to update indirect calls.
+ updateIndirectCalls(Index);
return;
+ }
unsigned LiveSymbols = 0;
SmallVector<ValueInfo, 128> Worklist;
Worklist.reserve(GUIDPreservedSymbols.size() * 2);
@@ -864,13 +873,16 @@ void llvm::computeDeadSymbols(
// Add values flagged in the index as live roots to the worklist.
for (const auto &Entry : Index) {
auto VI = Index.getValueInfo(Entry);
- for (auto &S : Entry.second.SummaryList)
+ for (auto &S : Entry.second.SummaryList) {
+ if (auto *FS = dyn_cast<FunctionSummary>(S.get()))
+ updateValueInfoForIndirectCalls(Index, FS);
if (S->isLive()) {
LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
Worklist.push_back(VI);
++LiveSymbols;
break;
}
+ }
}
// Make value live and add it to the worklist if it was not live before.
@@ -883,9 +895,6 @@ void llvm::computeDeadSymbols(
// binary, which increases the binary size unnecessarily. Note that
// if this code changes, the importer needs to change so that edges
// to functions marked dead are skipped.
- VI = updateValueInfoForIndirectCalls(Index, VI);
- if (!VI)
- return;
if (llvm::any_of(VI.getSummaryList(),
[](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
@@ -959,7 +968,8 @@ void llvm::computeDeadSymbolsWithConstProp(
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
bool ImportEnabled) {
- computeDeadSymbols(Index, GUIDPreservedSymbols, isPrevailing);
+ computeDeadSymbolsAndUpdateIndirectCalls(Index, GUIDPreservedSymbols,
+ isPrevailing);
if (ImportEnabled)
Index.propagateAttributes(GUIDPreservedSymbols);
}
diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp
index 5332ef28d94e7..45cca0f976118 100644
--- a/llvm/tools/llvm-lto/llvm-lto.cpp
+++ b/llvm/tools/llvm-lto/llvm-lto.cpp
@@ -487,6 +487,10 @@ static void createCombinedModuleSummaryIndex() {
ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename)));
ExitOnErr(readModuleSummaryIndex(*MB, CombinedIndex, NextModuleId++));
}
+ // In order to use this index for testing, specifically import testing, we
+ // need to update any indirect call edges created from SamplePGO, so that they
+ // point to the correct GUIDs.
+ updateIndirectCalls(CombinedIndex);
std::error_code EC;
assert(!OutputFilename.empty());
raw_fd_ostream OS(OutputFilename + ".thinlto.bc", EC,
More information about the llvm-commits
mailing list