[llvm] r340650 - Reduce the memory footprint of dsymutil. (NFC)
Adrian Prantl via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 24 13:41:08 PDT 2018
Author: adrian
Date: Fri Aug 24 13:41:08 2018
New Revision: 340650
URL: http://llvm.org/viewvc/llvm-project?rev=340650&view=rev
Log:
Reduce the memory footprint of dsymutil. (NFC)
This (partially) fixes a regression introduced by
https://reviews.llvm.org/D43945 / r327399, which parallelized
DwarfLinker. This patch avoids parsing and allocating the memory for
all input DIEs up front and instead only allocates them in the
concurrent loop in the AnalyzeLambda. At the end of the loop the
memory from the LinkContext is cleared again.
This reduces the peak memory needed to link the debug info of a
non-modular build of the Swift compiler by >3GB.
rdar://problem/43444464
Differential Revision: https://reviews.llvm.org/D51078
Modified:
llvm/trunk/tools/dsymutil/DwarfLinker.cpp
llvm/trunk/tools/dsymutil/DwarfLinker.h
Modified: llvm/trunk/tools/dsymutil/DwarfLinker.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/dsymutil/DwarfLinker.cpp?rev=340650&r1=340649&r2=340650&view=diff
==============================================================================
--- llvm/trunk/tools/dsymutil/DwarfLinker.cpp (original)
+++ llvm/trunk/tools/dsymutil/DwarfLinker.cpp Fri Aug 24 13:41:08 2018
@@ -2024,7 +2024,7 @@ bool DwarfLinker::registerModuleReferenc
const DWARFDie &CUDie, const DWARFUnit &Unit, DebugMap &ModuleMap,
const DebugMapObject &DMO, RangesTy &Ranges, OffsetsStringPool &StringPool,
UniquingStringPool &UniquingStringPool, DeclContextTree &ODRContexts,
- unsigned &UnitID, unsigned Indent) {
+ unsigned &UnitID, unsigned Indent, bool Quiet) {
std::string PCMfile = dwarf::toString(
CUDie.find({dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}), "");
if (PCMfile.empty())
@@ -2036,11 +2036,12 @@ bool DwarfLinker::registerModuleReferenc
std::string Name = dwarf::toString(CUDie.find(dwarf::DW_AT_name), "");
if (Name.empty()) {
- reportWarning("Anonymous module skeleton CU for " + PCMfile, DMO);
+ if (!Quiet)
+ reportWarning("Anonymous module skeleton CU for " + PCMfile, DMO);
return true;
}
- if (Options.Verbose) {
+ if (!Quiet && Options.Verbose) {
outs().indent(Indent);
outs() << "Found clang module reference " << PCMfile;
}
@@ -2050,16 +2051,16 @@ bool DwarfLinker::registerModuleReferenc
// FIXME: Until PR27449 (https://llvm.org/bugs/show_bug.cgi?id=27449) is
// fixed in clang, only warn about DWO_id mismatches in verbose mode.
// ASTFileSignatures will change randomly when a module is rebuilt.
- if (Options.Verbose && (Cached->second != DwoId))
+ if (!Quiet && Options.Verbose && (Cached->second != DwoId))
reportWarning(Twine("hash mismatch: this object file was built against a "
"different version of the module ") +
PCMfile,
DMO);
- if (Options.Verbose)
+ if (!Quiet && Options.Verbose)
outs() << " [cached].\n";
return true;
}
- if (Options.Verbose)
+ if (!Quiet && Options.Verbose)
outs() << " ...\n";
// Cyclic dependencies are disallowed by Clang, but we still
@@ -2067,7 +2068,7 @@ bool DwarfLinker::registerModuleReferenc
ClangModules.insert({PCMfile, DwoId});
if (Error E = loadClangModule(PCMfile, PCMpath, Name, DwoId, ModuleMap, DMO,
Ranges, StringPool, UniquingStringPool,
- ODRContexts, UnitID, Indent + 2)) {
+ ODRContexts, UnitID, Indent + 2, Quiet)) {
consumeError(std::move(E));
return false;
}
@@ -2096,14 +2097,12 @@ DwarfLinker::loadObject(const DebugMapOb
return *Object;
}
-Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath,
- StringRef ModuleName, uint64_t DwoId,
- DebugMap &ModuleMap,
- const DebugMapObject &DMO, RangesTy &Ranges,
- OffsetsStringPool &StringPool,
- UniquingStringPool &UniquingStringPool,
- DeclContextTree &ODRContexts,
- unsigned &UnitID, unsigned Indent) {
+Error DwarfLinker::loadClangModule(
+ StringRef Filename, StringRef ModulePath, StringRef ModuleName,
+ uint64_t DwoId, DebugMap &ModuleMap, const DebugMapObject &DMO,
+ RangesTy &Ranges, OffsetsStringPool &StringPool,
+ UniquingStringPool &UniquingStringPool, DeclContextTree &ODRContexts,
+ unsigned &UnitID, unsigned Indent, bool Quiet) {
SmallString<80> Path(Options.PrependPath);
if (sys::path::is_relative(Filename))
sys::path::append(Path, ModulePath, Filename);
@@ -2165,7 +2164,7 @@ Error DwarfLinker::loadClangModule(Strin
continue;
if (!registerModuleReference(CUDie, *CU, ModuleMap, DMO, Ranges, StringPool,
UniquingStringPool, ODRContexts, UnitID,
- Indent)) {
+ Indent, Quiet)) {
if (Unit) {
std::string Err =
(Filename +
@@ -2179,7 +2178,7 @@ Error DwarfLinker::loadClangModule(Strin
// ASTFileSignatures will change randomly when a module is rebuilt.
uint64_t PCMDwoId = getDwoId(CUDie, *CU);
if (PCMDwoId != DwoId) {
- if (Options.Verbose)
+ if (!Quiet && Options.Verbose)
reportWarning(
Twine("hash mismatch: this object file was built against a "
"different version of the module ") +
@@ -2201,7 +2200,7 @@ Error DwarfLinker::loadClangModule(Strin
}
if (!Unit->getOrigUnit().getUnitDIE().hasChildren())
return Error::success();
- if (Options.Verbose) {
+ if (!Quiet && Options.Verbose) {
outs().indent(Indent);
outs() << "cloning .debug_info from " << Filename << "\n";
}
@@ -2466,14 +2465,10 @@ bool DwarfLinker::link(const DebugMap &M
DumpOpts.Verbose = Options.Verbose;
CUDie.dump(outs(), 0, DumpOpts);
}
-
- if (!CUDie || LLVM_UNLIKELY(Options.Update) ||
- !registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
- LinkContext.Ranges, OffsetsStringPool,
- UniquingStringPool, ODRContexts, UnitID)) {
- LinkContext.CompileUnits.push_back(llvm::make_unique<CompileUnit>(
- *CU, UnitID++, !Options.NoODR && !Options.Update, ""));
- }
+ if (CUDie && !LLVM_UNLIKELY(Options.Update))
+ registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
+ LinkContext.Ranges, OffsetsStringPool,
+ UniquingStringPool, ODRContexts, UnitID);
}
}
@@ -2492,13 +2487,32 @@ bool DwarfLinker::link(const DebugMap &M
for (unsigned i = 0, e = NumObjects; i != e; ++i) {
auto &LinkContext = ObjectContexts[i];
- if (!LinkContext.ObjectFile) {
+ if (!LinkContext.ObjectFile || !LinkContext.DwarfContext) {
std::unique_lock<std::mutex> LockGuard(ProcessedFilesMutex);
ProcessedFiles.set(i);
ProcessedFilesConditionVariable.notify_one();
continue;
}
+ for (const auto &CU : LinkContext.DwarfContext->compile_units()) {
+ updateDwarfVersion(CU->getVersion());
+ // The !registerModuleReference() condition effectively skips
+ // over fully resolved skeleton units. This second pass of
+ // registerModuleReferences doesn't do any new work, but it
+ // will collect top-level errors, which are suppressed. Module
+ // warnings were already displayed in the first iteration.
+ bool Quiet = true;
+ auto CUDie = CU->getUnitDIE(false);
+ if (!CUDie || LLVM_UNLIKELY(Options.Update) ||
+ !registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
+ LinkContext.Ranges, OffsetsStringPool,
+ UniquingStringPool, ODRContexts, UnitID,
+ Quiet)) {
+ LinkContext.CompileUnits.push_back(llvm::make_unique<CompileUnit>(
+ *CU, UnitID++, !Options.NoODR && !Options.Update, ""));
+ }
+ }
+
// Now build the DIE parent links that we will use during the next phase.
for (auto &CurrentUnit : LinkContext.CompileUnits) {
auto CUDie = CurrentUnit->getOrigUnit().getUnitDIE();
Modified: llvm/trunk/tools/dsymutil/DwarfLinker.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/dsymutil/DwarfLinker.h?rev=340650&r1=340649&r2=340650&view=diff
==============================================================================
--- llvm/trunk/tools/dsymutil/DwarfLinker.h (original)
+++ llvm/trunk/tools/dsymutil/DwarfLinker.h Fri Aug 24 13:41:08 2018
@@ -200,7 +200,7 @@ private:
OffsetsStringPool &OffsetsStringPool,
UniquingStringPool &UniquingStringPoolStringPool,
DeclContextTree &ODRContexts, unsigned &UnitID,
- unsigned Indent = 0);
+ unsigned Indent = 0, bool Quiet = false);
/// Recursively add the debug info in this clang module .pcm
/// file (and all the modules imported by it in a bottom-up fashion)
@@ -211,7 +211,7 @@ private:
RangesTy &Ranges, OffsetsStringPool &OffsetsStringPool,
UniquingStringPool &UniquingStringPool,
DeclContextTree &ODRContexts, unsigned &UnitID,
- unsigned Indent = 0);
+ unsigned Indent = 0, bool Quiet = false);
/// Flags passed to DwarfLinker::lookForDIEsToKeep
enum TraversalFlags {
More information about the llvm-commits
mailing list