[lld] r357383 - [LLD][COFF] Early dependency detection
Alexandre Ganea via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 1 06:36:59 PDT 2019
Author: aganea
Date: Mon Apr 1 06:36:59 2019
New Revision: 357383
URL: http://llvm.org/viewvc/llvm-project?rev=357383&view=rev
Log:
[LLD][COFF] Early dependency detection
We introduce a new class hierarchy for debug types merging (in DebugTypes.h). The end-goal is to parallelize the type merging - please see the plan in D59226.
Previously, dependency discovery was done on the fly, much later, during the type merging loop. Unfortunately, parallelizing the type merging requires the dependencies to be merged in first, before any dependent ObjFile, thus this early discovery.
The overall intention for this path is to discover debug information dependencies at a much earlier stage, when processing input files. Currently, two types of dependency are supported: PDB type servers (when compiling with MSVC /Zi) and precompiled headers OBJs (when compiling with MSVC /Yc and /Yu). Once discovered, an explicit link is added into the dependent ObjFile, through the new debug types class hierarchy introduced in DebugTypes.h.
Differential Revision: https://reviews.llvm.org/D59053
Added:
lld/trunk/COFF/DebugTypes.cpp
lld/trunk/COFF/DebugTypes.h
Modified:
lld/trunk/COFF/CMakeLists.txt
lld/trunk/COFF/InputFiles.cpp
lld/trunk/COFF/InputFiles.h
lld/trunk/COFF/PDB.cpp
Modified: lld/trunk/COFF/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/CMakeLists.txt?rev=357383&r1=357382&r2=357383&view=diff
==============================================================================
--- lld/trunk/COFF/CMakeLists.txt (original)
+++ lld/trunk/COFF/CMakeLists.txt Mon Apr 1 06:36:59 2019
@@ -8,6 +8,7 @@ endif()
add_lld_library(lldCOFF
Chunks.cpp
+ DebugTypes.cpp
DLL.cpp
Driver.cpp
DriverUtils.cpp
Added: lld/trunk/COFF/DebugTypes.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/DebugTypes.cpp?rev=357383&view=auto
==============================================================================
--- lld/trunk/COFF/DebugTypes.cpp (added)
+++ lld/trunk/COFF/DebugTypes.cpp Mon Apr 1 06:36:59 2019
@@ -0,0 +1,84 @@
+//===- DebugTypes.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DebugTypes.h"
+#include "InputFiles.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+
+using namespace lld;
+using namespace lld::coff;
+using namespace llvm;
+using namespace llvm::codeview;
+
+namespace {
+class TypeServerSource : public TpiSource {
+public:
+ TypeServerSource(ObjFile *F) : TpiSource(PDB, F) {}
+};
+
+class UseTypeServerSource : public TpiSource {
+public:
+ UseTypeServerSource(ObjFile *F, TypeServer2Record *TS)
+ : TpiSource(UsingPDB, F), TypeServerDependency(*TS) {}
+
+ // Information about the PDB type server dependency, that needs to be loaded
+ // in before merging this OBJ.
+ TypeServer2Record TypeServerDependency;
+};
+
+class PrecompSource : public TpiSource {
+public:
+ PrecompSource(ObjFile *F) : TpiSource(PCH, F) {}
+};
+
+class UsePrecompSource : public TpiSource {
+public:
+ UsePrecompSource(ObjFile *F, PrecompRecord *Precomp)
+ : TpiSource(UsingPCH, F), PrecompDependency(*Precomp) {}
+
+ // Information about the Precomp OBJ dependency, that needs to be loaded in
+ // before merging this OBJ.
+ PrecompRecord PrecompDependency;
+};
+} // namespace
+
+static std::vector<std::unique_ptr<TpiSource>> GC;
+
+TpiSource::TpiSource(TpiKind K, ObjFile *F) : Kind(K), File(F) {
+ GC.push_back(std::unique_ptr<TpiSource>(this));
+}
+
+TpiSource *coff::makeTpiSource(ObjFile *F) {
+ return new TpiSource(TpiSource::Regular, F);
+}
+
+TpiSource *coff::makeTypeServerSource(ObjFile *F) {
+ return new TypeServerSource(F);
+}
+
+TpiSource *coff::makeUseTypeServerSource(ObjFile *F, TypeServer2Record *TS) {
+ return new UseTypeServerSource(F, TS);
+}
+
+TpiSource *coff::makePrecompSource(ObjFile *F) { return new PrecompSource(F); }
+
+TpiSource *coff::makeUsePrecompSource(ObjFile *F, PrecompRecord *Precomp) {
+ return new UsePrecompSource(F, Precomp);
+}
+
+template <>
+const PrecompRecord &coff::retrieveDependencyInfo(TpiSource *Source) {
+ assert(Source->Kind == TpiSource::UsingPCH);
+ return ((UsePrecompSource *)Source)->PrecompDependency;
+}
+
+template <>
+const TypeServer2Record &coff::retrieveDependencyInfo(TpiSource *Source) {
+ assert(Source->Kind == TpiSource::UsingPDB);
+ return ((UseTypeServerSource *)Source)->TypeServerDependency;
+}
\ No newline at end of file
Added: lld/trunk/COFF/DebugTypes.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/DebugTypes.h?rev=357383&view=auto
==============================================================================
--- lld/trunk/COFF/DebugTypes.h (added)
+++ lld/trunk/COFF/DebugTypes.h Mon Apr 1 06:36:59 2019
@@ -0,0 +1,51 @@
+//===- DebugTypes.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_COFF_DEBUGTYPES_H
+#define LLD_COFF_DEBUGTYPES_H
+
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace codeview {
+class PrecompRecord;
+class TypeServer2Record;
+} // namespace codeview
+} // namespace llvm
+
+namespace lld {
+namespace coff {
+
+class ObjFile;
+
+class TpiSource {
+public:
+ enum TpiKind { Regular, PCH, UsingPCH, PDB, UsingPDB };
+
+ TpiSource(TpiKind K, ObjFile *F);
+ virtual ~TpiSource() {}
+
+ const TpiKind Kind;
+ ObjFile *File;
+};
+
+TpiSource *makeTpiSource(ObjFile *F);
+TpiSource *makeTypeServerSource(ObjFile *F);
+TpiSource *makeUseTypeServerSource(ObjFile *F,
+ llvm::codeview::TypeServer2Record *TS);
+TpiSource *makePrecompSource(ObjFile *F);
+TpiSource *makeUsePrecompSource(ObjFile *F,
+ llvm::codeview::PrecompRecord *Precomp);
+
+// Temporary interface to get the dependency
+template <typename T> const T &retrieveDependencyInfo(TpiSource *Source);
+
+} // namespace coff
+} // namespace lld
+
+#endif
\ No newline at end of file
Modified: lld/trunk/COFF/InputFiles.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/InputFiles.cpp?rev=357383&r1=357382&r2=357383&view=diff
==============================================================================
--- lld/trunk/COFF/InputFiles.cpp (original)
+++ lld/trunk/COFF/InputFiles.cpp Mon Apr 1 06:36:59 2019
@@ -9,6 +9,7 @@
#include "InputFiles.h"
#include "Chunks.h"
#include "Config.h"
+#include "DebugTypes.h"
#include "Driver.h"
#include "SymbolTable.h"
#include "Symbols.h"
@@ -22,6 +23,7 @@
#include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h"
#include "llvm/DebugInfo/CodeView/SymbolDeserializer.h"
#include "llvm/DebugInfo/CodeView/SymbolRecord.h"
+#include "llvm/DebugInfo/CodeView/TypeDeserializer.h"
#include "llvm/Object/Binary.h"
#include "llvm/Object/COFF.h"
#include "llvm/Support/Casting.h"
@@ -129,6 +131,7 @@ void ObjFile::parse() {
initializeChunks();
initializeSymbols();
initializeFlags();
+ initializeDependencies();
}
const coff_section* ObjFile::getSection(uint32_t I) {
@@ -656,6 +659,59 @@ void ObjFile::initializeFlags() {
}
}
+// Depending on the compilation flags, OBJs can refer to external files,
+// necessary to merge this OBJ into the final PDB. We currently support two
+// types of external files: Precomp/PCH OBJs, when compiling with /Yc and /Yu.
+// And PDB type servers, when compiling with /Zi. This function extracts these
+// dependencies and makes them available as a TpiSource interface (see
+// DebugTypes.h).
+void ObjFile::initializeDependencies() {
+ if (!Config->Debug)
+ return;
+
+ bool IsPCH = false;
+
+ ArrayRef<uint8_t> Data = getDebugSection(".debug$P");
+ if (!Data.empty())
+ IsPCH = true;
+ else
+ Data = getDebugSection(".debug$T");
+
+ if (Data.empty())
+ return;
+
+ CVTypeArray Types;
+ BinaryStreamReader Reader(Data, support::little);
+ cantFail(Reader.readArray(Types, Reader.getLength()));
+
+ CVTypeArray::Iterator FirstType = Types.begin();
+ if (FirstType == Types.end())
+ return;
+
+ DebugTypes.emplace(Types);
+
+ if (IsPCH) {
+ DebugTypesObj = makePrecompSource(this);
+ return;
+ }
+
+ if (FirstType->kind() == LF_TYPESERVER2) {
+ TypeServer2Record TS = cantFail(
+ TypeDeserializer::deserializeAs<TypeServer2Record>(FirstType->data()));
+ DebugTypesObj = makeUseTypeServerSource(this, &TS);
+ return;
+ }
+
+ if (FirstType->kind() == LF_PRECOMP) {
+ PrecompRecord Precomp = cantFail(
+ TypeDeserializer::deserializeAs<PrecompRecord>(FirstType->data()));
+ DebugTypesObj = makeUsePrecompSource(this, &Precomp);
+ return;
+ }
+
+ DebugTypesObj = makeTpiSource(this);
+}
+
StringRef ltrim1(StringRef S, const char *Chars) {
if (!S.empty() && strchr(Chars, S[0]))
return S.substr(1);
Modified: lld/trunk/COFF/InputFiles.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/InputFiles.h?rev=357383&r1=357382&r2=357383&view=diff
==============================================================================
--- lld/trunk/COFF/InputFiles.h (original)
+++ lld/trunk/COFF/InputFiles.h Mon Apr 1 06:36:59 2019
@@ -51,6 +51,7 @@ class Lazy;
class SectionChunk;
class Symbol;
class Undefined;
+class TpiSource;
// The root class of input files.
class InputFile {
@@ -167,6 +168,12 @@ public:
// Whether the object was already merged into the final PDB or not
bool MergedIntoPDB = false;
+ // If the OBJ has a .debug$T stream, this tells how it will be handled.
+ TpiSource *DebugTypesObj = nullptr;
+
+ // The .debug$T stream if there's one.
+ llvm::Optional<llvm::codeview::CVTypeArray> DebugTypes;
+
private:
const coff_section* getSection(uint32_t I);
const coff_section *getSection(COFFSymbolRef Sym) {
@@ -176,6 +183,7 @@ private:
void initializeChunks();
void initializeSymbols();
void initializeFlags();
+ void initializeDependencies();
SectionChunk *
readSection(uint32_t SectionNumber,
Modified: lld/trunk/COFF/PDB.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/PDB.cpp?rev=357383&r1=357382&r2=357383&view=diff
==============================================================================
--- lld/trunk/COFF/PDB.cpp (original)
+++ lld/trunk/COFF/PDB.cpp Mon Apr 1 06:36:59 2019
@@ -9,6 +9,7 @@
#include "PDB.h"
#include "Chunks.h"
#include "Config.h"
+#include "DebugTypes.h"
#include "Driver.h"
#include "SymbolTable.h"
#include "Symbols.h"
@@ -132,15 +133,12 @@ public:
CVIndexMap *ObjectIndexMap);
/// Reads and makes available a PDB.
- Expected<const CVIndexMap &> maybeMergeTypeServerPDB(ObjFile *File,
- const CVType &FirstType);
+ Expected<const CVIndexMap &> maybeMergeTypeServerPDB(ObjFile *File);
/// Merges a precompiled headers TPI map into the current TPI map. The
/// precompiled headers object will also be loaded and remapped in the
/// process.
- Expected<const CVIndexMap &>
- mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType,
- CVIndexMap *ObjectIndexMap);
+ Error mergeInPrecompHeaderObj(ObjFile *File, CVIndexMap *ObjectIndexMap);
/// Reads and makes available a precompiled headers object.
///
@@ -151,8 +149,7 @@ public:
///
/// If the precompiled headers object was already loaded, this function will
/// simply return its (remapped) TPI map.
- Expected<const CVIndexMap &> aquirePrecompObj(ObjFile *File,
- PrecompRecord Precomp);
+ Expected<const CVIndexMap &> aquirePrecompObj(ObjFile *File);
/// Adds a precompiled headers object signature -> TPI mapping.
std::pair<CVIndexMap &, bool /*already there*/>
@@ -373,21 +370,12 @@ Expected<const CVIndexMap &>
PDBLinker::mergeDebugT(ObjFile *File, CVIndexMap *ObjectIndexMap) {
ScopedTimer T(TypeMergingTimer);
- bool IsPrecompiledHeader = false;
-
- ArrayRef<uint8_t> Data = File->getDebugSection(".debug$T");
- if (Data.empty()) {
- // Try again, Microsoft precompiled headers use .debug$P instead of
- // .debug$T
- Data = File->getDebugSection(".debug$P");
- IsPrecompiledHeader = true;
- }
- if (Data.empty())
- return *ObjectIndexMap; // no debug info
+ if (!File->DebugTypesObj)
+ return *ObjectIndexMap; // no Types stream
// Precompiled headers objects need to save the index map for further
// reference by other objects which use the precompiled headers.
- if (IsPrecompiledHeader) {
+ if (File->DebugTypesObj->Kind == TpiSource::PCH) {
uint32_t PCHSignature = File->PCHSignature.getValueOr(0);
if (PCHSignature == 0)
fatal("No signature found for the precompiled headers OBJ (" +
@@ -409,33 +397,28 @@ PDBLinker::mergeDebugT(ObjFile *File, CV
}
}
- BinaryByteStream Stream(Data, support::little);
- CVTypeArray Types;
- BinaryStreamReader Reader(Stream);
- if (auto EC = Reader.readArray(Types, Reader.getLength()))
- fatal("Reader::readArray failed: " + toString(std::move(EC)));
-
- auto FirstType = Types.begin();
- if (FirstType == Types.end())
- return *ObjectIndexMap;
-
- if (FirstType->kind() == LF_TYPESERVER2) {
+ if (File->DebugTypesObj->Kind == TpiSource::UsingPDB) {
// Look through type servers. If we've already seen this type server,
// don't merge any type information.
- return maybeMergeTypeServerPDB(File, *FirstType);
- } else if (FirstType->kind() == LF_PRECOMP) {
+ return maybeMergeTypeServerPDB(File);
+ }
+
+ CVTypeArray &Types = *File->DebugTypes;
+
+ if (File->DebugTypesObj->Kind == TpiSource::UsingPCH) {
// This object was compiled with /Yu, so process the corresponding
// precompiled headers object (/Yc) first. Some type indices in the current
// object are referencing data in the precompiled headers object, so we need
// both to be loaded.
- auto E = mergeInPrecompHeaderObj(File, *FirstType, ObjectIndexMap);
- if (!E)
- return E.takeError();
-
- // Drop LF_PRECOMP record from the input stream, as it needs to be replaced
- // with the precompiled headers object type stream.
- // Note that we can't just call Types.drop_front(), as we explicitly want to
- // rebase the stream.
+ Error E = mergeInPrecompHeaderObj(File, ObjectIndexMap);
+ if (E)
+ return std::move(E);
+
+ // Drop LF_PRECOMP record from the input stream, as it has been replaced
+ // with the precompiled headers Type stream in the mergeInPrecompHeaderObj()
+ // call above. Note that we can't just call Types.drop_front(), as we
+ // explicitly want to rebase the stream.
+ CVTypeArray::Iterator FirstType = Types.begin();
Types.setUnderlyingStream(
Types.getUnderlyingStream().drop_front(FirstType->RecordData.size()));
}
@@ -503,12 +486,9 @@ tryToLoadPDB(const codeview::GUID &GuidF
return std::move(NS);
}
-Expected<const CVIndexMap &>
-PDBLinker::maybeMergeTypeServerPDB(ObjFile *File, const CVType &FirstType) {
- TypeServer2Record TS;
- if (auto EC =
- TypeDeserializer::deserializeAs(const_cast<CVType &>(FirstType), TS))
- fatal("error reading record: " + toString(std::move(EC)));
+Expected<const CVIndexMap &> PDBLinker::maybeMergeTypeServerPDB(ObjFile *File) {
+ const TypeServer2Record &TS =
+ retrieveDependencyInfo<TypeServer2Record>(File->DebugTypesObj);
const codeview::GUID &TSId = TS.getGuid();
StringRef TSPath = TS.getName();
@@ -617,15 +597,12 @@ PDBLinker::maybeMergeTypeServerPDB(ObjFi
return IndexMap;
}
-Expected<const CVIndexMap &>
-PDBLinker::mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType,
- CVIndexMap *ObjectIndexMap) {
- PrecompRecord Precomp;
- if (auto EC = TypeDeserializer::deserializeAs(const_cast<CVType &>(FirstType),
- Precomp))
- fatal("error reading record: " + toString(std::move(EC)));
+Error PDBLinker::mergeInPrecompHeaderObj(ObjFile *File,
+ CVIndexMap *ObjectIndexMap) {
+ const PrecompRecord &Precomp =
+ retrieveDependencyInfo<PrecompRecord>(File->DebugTypesObj);
- auto E = aquirePrecompObj(File, Precomp);
+ Expected<const CVIndexMap &> E = aquirePrecompObj(File);
if (!E)
return E.takeError();
@@ -633,7 +610,7 @@ PDBLinker::mergeInPrecompHeaderObj(ObjFi
assert(PrecompIndexMap.IsPrecompiledTypeMap);
if (PrecompIndexMap.TPIMap.empty())
- return PrecompIndexMap;
+ return Error::success();
assert(Precomp.getStartTypeIndex() == TypeIndex::FirstNonSimpleIndex);
assert(Precomp.getTypesCount() <= PrecompIndexMap.TPIMap.size());
@@ -641,7 +618,7 @@ PDBLinker::mergeInPrecompHeaderObj(ObjFi
ObjectIndexMap->TPIMap.append(PrecompIndexMap.TPIMap.begin(),
PrecompIndexMap.TPIMap.begin() +
Precomp.getTypesCount());
- return *ObjectIndexMap;
+ return Error::success();
}
static bool equals_path(StringRef path1, StringRef path2) {
@@ -677,8 +654,10 @@ PDBLinker::registerPrecompiledHeaders(ui
return {IndexMap, false};
}
-Expected<const CVIndexMap &>
-PDBLinker::aquirePrecompObj(ObjFile *File, PrecompRecord Precomp) {
+Expected<const CVIndexMap &> PDBLinker::aquirePrecompObj(ObjFile *File) {
+ const PrecompRecord &Precomp =
+ retrieveDependencyInfo<PrecompRecord>(File->DebugTypesObj);
+
// First, check if we already loaded the precompiled headers object with this
// signature. Return the type index mapping if we've already seen it.
auto R = registerPrecompiledHeaders(Precomp.getSignature());
More information about the llvm-commits
mailing list