[lld] r346154 - [COFF][LLD] Add link support for Microsoft precompiled headers OBJs
Alexandre Ganea via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 5 11:20:47 PST 2018
Author: aganea
Date: Mon Nov 5 11:20:47 2018
New Revision: 346154
URL: http://llvm.org/viewvc/llvm-project?rev=346154&view=rev
Log:
[COFF][LLD] Add link support for Microsoft precompiled headers OBJs
This change allows for link-time merging of debugging information from
Microsoft precompiled types OBJs compiled with cl.exe /Z7 /Yc and /Yu.
This fixes llvm.org/PR34278
Differential Revision: https://reviews.llvm.org/D45213
Added:
lld/trunk/test/COFF/Inputs/precomp-a.obj (with props)
lld/trunk/test/COFF/Inputs/precomp-b.obj (with props)
lld/trunk/test/COFF/Inputs/precomp-invalid.obj (with props)
lld/trunk/test/COFF/Inputs/precomp.obj (with props)
lld/trunk/test/COFF/precomp-link.test
Modified:
lld/trunk/COFF/InputFiles.h
lld/trunk/COFF/PDB.cpp
Modified: lld/trunk/COFF/InputFiles.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/InputFiles.h?rev=346154&r1=346153&r2=346154&view=diff
==============================================================================
--- lld/trunk/COFF/InputFiles.h (original)
+++ lld/trunk/COFF/InputFiles.h Mon Nov 5 11:20:47 2018
@@ -15,6 +15,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/Archive.h"
#include "llvm/Object/COFF.h"
@@ -122,9 +123,12 @@ public:
return Symbols[SymbolIndex];
}
- // Returns the underying COFF file.
+ // Returns the underlying COFF file.
COFFObjectFile *getCOFFObj() { return COFFObj.get(); }
+ // Whether the object was already merged into the final PDB or not
+ bool wasProcessedForPDB() const { return !!ModuleDBI; }
+
static std::vector<ObjFile *> Instances;
// Flags in the absolute @feat.00 symbol if it is present. These usually
@@ -147,6 +151,11 @@ public:
const coff_section *AddrsigSec = nullptr;
+ // When using Microsoft precompiled headers, this is the PCH's key.
+ // The same key is used by both the precompiled object, and objects using the
+ // precompiled object. Any difference indicates out-of-date objects.
+ llvm::Optional<llvm::codeview::EndPrecompRecord> EndPrecomp;
+
private:
void initializeChunks();
void initializeSymbols();
Modified: lld/trunk/COFF/PDB.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/PDB.cpp?rev=346154&r1=346153&r2=346154&view=diff
==============================================================================
--- lld/trunk/COFF/PDB.cpp (original)
+++ lld/trunk/COFF/PDB.cpp Mon Nov 5 11:20:47 2018
@@ -49,6 +49,7 @@
#include "llvm/Object/CVDebugRecord.h"
#include "llvm/Support/BinaryByteStream.h"
#include "llvm/Support/Endian.h"
+#include "llvm/Support/Errc.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/JamCRC.h"
#include "llvm/Support/Path.h"
@@ -80,6 +81,7 @@ struct CVIndexMap {
SmallVector<TypeIndex, 0> TPIMap;
SmallVector<TypeIndex, 0> IPIMap;
bool IsTypeServerMap = false;
+ bool IsPrecompiledTypeMap = false;
};
class DebugSHandler;
@@ -106,8 +108,10 @@ public:
/// Link CodeView from each object file in the symbol table into the PDB.
void addObjectsToPDB();
- /// Link CodeView from a single object file into the PDB.
- void addObjFile(ObjFile *File);
+ /// Link CodeView from a single object file into the target (output) PDB.
+ /// When a precompiled headers object is linked, its TPI map might be provided
+ /// externally.
+ void addObjFile(ObjFile *File, CVIndexMap *ExternIndexMap = nullptr);
/// Produce a mapping from the type and item indices used in the object
/// file to those in the destination PDB.
@@ -120,11 +124,35 @@ public:
/// If the object does not use a type server PDB (compiled with /Z7), we merge
/// all the type and item records from the .debug$S stream and fill in the
/// caller-provided ObjectIndexMap.
- Expected<const CVIndexMap&> mergeDebugT(ObjFile *File,
- CVIndexMap &ObjectIndexMap);
+ Expected<const CVIndexMap &> mergeDebugT(ObjFile *File,
+ CVIndexMap *ObjectIndexMap);
- Expected<const CVIndexMap&> maybeMergeTypeServerPDB(ObjFile *File,
- TypeServer2Record &TS);
+ /// Reads and makes available a PDB.
+ Expected<const CVIndexMap &> maybeMergeTypeServerPDB(ObjFile *File,
+ const CVType &FirstType);
+
+ /// Merges a precompiled headers TPI map into the current TPI map. The
+ /// precompiled headers object will also be loaded and remapped in the
+ /// process.
+ Expected<const CVIndexMap &>
+ mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType,
+ CVIndexMap *ObjectIndexMap);
+
+ /// Reads and makes available a precompiled headers object.
+ ///
+ /// This is a requirement for objects compiled with cl.exe /Yu. In that
+ /// case, the referenced object (which was compiled with /Yc) has to be loaded
+ /// first. This is mainly because the current object's TPI stream has external
+ /// references to the precompiled headers object.
+ ///
+ /// If the precompiled headers object was already loaded, this function will
+ /// simply return its (remapped) TPI map.
+ Expected<const CVIndexMap &> aquirePrecompObj(ObjFile *File,
+ PrecompRecord Precomp);
+
+ /// Adds a precompiled headers object signature -> TPI mapping.
+ std::pair<CVIndexMap &, bool /*already there*/>
+ registerPrecompiledHeaders(uint32_t Signature);
/// Add the section map and section contributions to the PDB.
void addSections(ArrayRef<OutputSection *> OutputSections,
@@ -168,6 +196,10 @@ private:
/// Type index mappings of type server PDBs that we've loaded so far.
std::map<GUID, CVIndexMap> TypeServerIndexMappings;
+ /// Type index mappings of precompiled objects type map that we've loaded so
+ /// far.
+ std::map<uint32_t, CVIndexMap> PrecompTypeIndexMappings;
+
/// List of TypeServer PDBs which cannot be loaded.
/// Cached to prevent repeated load attempts.
std::map<GUID, std::string> MissingTypeServerPDBs;
@@ -324,27 +356,79 @@ static void addTypeInfo(pdb::TpiStreamBu
});
}
-static Optional<TypeServer2Record>
-maybeReadTypeServerRecord(CVTypeArray &Types) {
- auto I = Types.begin();
- if (I == Types.end())
- return None;
- const CVType &Type = *I;
- if (Type.kind() != LF_TYPESERVER2)
- return None;
- TypeServer2Record TS;
- if (auto EC = TypeDeserializer::deserializeAs(const_cast<CVType &>(Type), TS))
- fatal("error reading type server record: " + toString(std::move(EC)));
- return std::move(TS);
+// OBJs usually start their symbol stream with a S_OBJNAME record. This record
+// also contains the signature/key of the current PCH session. The signature
+// must be same for all objects which depend on the precompiled object.
+// Recompiling the precompiled headers will generate a new PCH key and thus
+// invalidate all the dependent objects.
+static uint32_t extractPCHSignature(ObjFile *File) {
+ auto DbgIt = find_if(File->getDebugChunks(), [](auto &C) {
+ return C->getSectionName() == ".debug$S";
+ });
+ if (!DbgIt)
+ return 0;
+
+ ArrayRef<uint8_t> Contents =
+ consumeDebugMagic((*DbgIt)->getContents(), ".debug$S");
+ DebugSubsectionArray Subsections;
+ BinaryStreamReader Reader(Contents, support::little);
+ ExitOnErr(Reader.readArray(Subsections, Contents.size()));
+
+ for (const DebugSubsectionRecord &SS : Subsections) {
+ if (SS.kind() != DebugSubsectionKind::Symbols)
+ continue;
+
+ // If it's there, the S_OBJNAME record shall come first in the stream.
+ Expected<CVSymbol> Sym = readSymbolFromStream(SS.getRecordData(), 0);
+ if (!Sym) {
+ consumeError(Sym.takeError());
+ continue;
+ }
+ if (auto ObjName = SymbolDeserializer::deserializeAs<ObjNameSym>(Sym.get()))
+ return ObjName->Signature;
+ }
+ return 0;
}
-Expected<const CVIndexMap&> PDBLinker::mergeDebugT(ObjFile *File,
- CVIndexMap &ObjectIndexMap) {
+Expected<const CVIndexMap &>
+PDBLinker::mergeDebugT(ObjFile *File, CVIndexMap *ObjectIndexMap) {
ScopedTimer T(TypeMergingTimer);
+ bool IsPrecompiledHeader = false;
+
ArrayRef<uint8_t> Data = getDebugSection(File, ".debug$T");
+ if (Data.empty()) {
+ // Try again, Microsoft precompiled headers use .debug$P instead of
+ // .debug$T
+ Data = getDebugSection(File, ".debug$P");
+ IsPrecompiledHeader = true;
+ }
if (Data.empty())
- return ObjectIndexMap;
+ return *ObjectIndexMap; // no debug info
+
+ // Precompiled headers objects need to save the index map for further
+ // reference by other objects which use the precompiled headers.
+ if (IsPrecompiledHeader) {
+ uint32_t PCHSignature = extractPCHSignature(File);
+ if (PCHSignature == 0)
+ fatal("No signature found for the precompiled headers OBJ (" +
+ File->getName() + ")");
+
+ // When a precompiled headers object comes first on the command-line, we
+ // update the mapping here. Otherwise, if an object referencing the
+ // precompiled headers object comes first, the mapping is created in
+ // aquirePrecompObj(), thus we would skip this block.
+ if (!ObjectIndexMap->IsPrecompiledTypeMap) {
+ auto R = registerPrecompiledHeaders(PCHSignature);
+ if (R.second)
+ fatal(
+ "A precompiled headers OBJ with the same signature was already "
+ "provided! (" +
+ File->getName() + ")");
+
+ ObjectIndexMap = &R.first;
+ }
+ }
BinaryByteStream Stream(Data, support::little);
CVTypeArray Types;
@@ -352,13 +436,29 @@ Expected<const CVIndexMap&> PDBLinker::m
if (auto EC = Reader.readArray(Types, Reader.getLength()))
fatal("Reader::readArray failed: " + toString(std::move(EC)));
- // Look through type servers. If we've already seen this type server, don't
- // merge any type information.
- if (Optional<TypeServer2Record> TS = maybeReadTypeServerRecord(Types))
- return maybeMergeTypeServerPDB(File, *TS);
+ auto FirstType = Types.begin();
+ if (FirstType == Types.end())
+ return *ObjectIndexMap;
+
+ if (FirstType->kind() == LF_TYPESERVER2) {
+ // Look through type servers. If we've already seen this type server,
+ // don't merge any type information.
+ return maybeMergeTypeServerPDB(File, *FirstType);
+ } else if (FirstType->kind() == LF_PRECOMP) {
+ // This object was compiled with /Yu, so process the corresponding
+ // precompiled headers object (/Yc) first. Some type indices in the current
+ // object are referencing data in the precompiled headers object, so we need
+ // both to be loaded.
+ auto E = mergeInPrecompHeaderObj(File, *FirstType, ObjectIndexMap);
+ if (!E)
+ return E.takeError();
+
+ // Drop LF_PRECOMP record from the input stream, as it needs to be replaced
+ // with the precompiled headers object type stream.
+ Types.drop_front();
+ }
- // This is a /Z7 object. Fill in the temporary, caller-provided
- // ObjectIndexMap.
+ // Fill in the temporary, caller-provided ObjectIndexMap.
if (Config->DebugGHashes) {
ArrayRef<GloballyHashedType> Hashes;
std::vector<GloballyHashedType> OwnedHashes;
@@ -370,16 +470,18 @@ Expected<const CVIndexMap&> PDBLinker::m
}
if (auto Err = mergeTypeAndIdRecords(GlobalIDTable, GlobalTypeTable,
- ObjectIndexMap.TPIMap, Types, Hashes))
+ ObjectIndexMap->TPIMap, Types, Hashes,
+ File->EndPrecomp))
fatal("codeview::mergeTypeAndIdRecords failed: " +
toString(std::move(Err)));
} else {
- if (auto Err = mergeTypeAndIdRecords(IDTable, TypeTable,
- ObjectIndexMap.TPIMap, Types))
+ if (auto Err =
+ mergeTypeAndIdRecords(IDTable, TypeTable, ObjectIndexMap->TPIMap,
+ Types, File->EndPrecomp))
fatal("codeview::mergeTypeAndIdRecords failed: " +
toString(std::move(Err)));
}
- return ObjectIndexMap;
+ return *ObjectIndexMap;
}
static Expected<std::unique_ptr<pdb::NativeSession>>
@@ -420,7 +522,12 @@ tryToLoadPDB(const GUID &GuidFromObj, St
}
Expected<const CVIndexMap &>
-PDBLinker::maybeMergeTypeServerPDB(ObjFile *File, TypeServer2Record &TS) {
+PDBLinker::maybeMergeTypeServerPDB(ObjFile *File, const CVType &FirstType) {
+ TypeServer2Record TS;
+ if (auto EC =
+ TypeDeserializer::deserializeAs(const_cast<CVType &>(FirstType), TS))
+ fatal("error reading record: " + toString(std::move(EC)));
+
const GUID &TSId = TS.getGuid();
StringRef TSPath = TS.getName();
@@ -502,9 +609,10 @@ PDBLinker::maybeMergeTypeServerPDB(ObjFi
auto IpiHashes =
GloballyHashedType::hashIds(ExpectedIpi->typeArray(), TpiHashes);
+ Optional<EndPrecompRecord> EndPrecomp;
// Merge TPI first, because the IPI stream will reference type indices.
if (auto Err = mergeTypeRecords(GlobalTypeTable, IndexMap.TPIMap,
- ExpectedTpi->typeArray(), TpiHashes))
+ ExpectedTpi->typeArray(), TpiHashes, EndPrecomp))
fatal("codeview::mergeTypeRecords failed: " + toString(std::move(Err)));
// Merge IPI.
@@ -527,6 +635,113 @@ PDBLinker::maybeMergeTypeServerPDB(ObjFi
return IndexMap;
}
+Expected<const CVIndexMap &>
+PDBLinker::mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType,
+ CVIndexMap *ObjectIndexMap) {
+ PrecompRecord Precomp;
+ if (auto EC = TypeDeserializer::deserializeAs(const_cast<CVType &>(FirstType),
+ Precomp))
+ fatal("error reading record: " + toString(std::move(EC)));
+
+ auto E = aquirePrecompObj(File, Precomp);
+ if (!E)
+ return createFileError(Precomp.getPrecompFilePath().str(), E.takeError());
+
+ const CVIndexMap &PrecompIndexMap = *E;
+ assert(PrecompIndexMap.IsPrecompiledTypeMap);
+
+ if (PrecompIndexMap.TPIMap.empty())
+ return PrecompIndexMap;
+
+ assert(Precomp.getStartTypeIndex() == TypeIndex::FirstNonSimpleIndex);
+ assert(Precomp.getTypesCount() <= PrecompIndexMap.TPIMap.size());
+ // Use the previously remapped index map from the precompiled headers.
+ ObjectIndexMap->TPIMap.append(PrecompIndexMap.TPIMap.begin(),
+ PrecompIndexMap.TPIMap.begin() +
+ Precomp.getTypesCount());
+ return *ObjectIndexMap;
+}
+
+static bool equals_path(StringRef path1, StringRef path2) {
+#if defined(_WIN32)
+ return path1.equals_lower(path2);
+#else
+ return path1.equals(path2);
+#endif
+}
+
+// Find an OBJ provided on the command line, either by name or full path
+static Optional<std::pair<ObjFile *, std::string>>
+findObjByName(StringRef NameOrPath) {
+ SmallString<128> CurrentPath;
+
+ StringRef FileNameOnly = sys::path::filename(NameOrPath);
+
+ for (ObjFile *F : ObjFile::Instances) {
+ CurrentPath = F->getName();
+ sys::fs::make_absolute(CurrentPath);
+
+ // First compare with the full path name
+ if (equals_path(CurrentPath, NameOrPath))
+ return std::make_pair(F, CurrentPath.str().str());
+
+ StringRef CurrentFileName = sys::path::filename(CurrentPath);
+
+ // Otherwise compare based solely on the file name (link.exe behavior)
+ if (equals_path(CurrentFileName, FileNameOnly))
+ return std::make_pair(F, CurrentPath.str().str());
+ }
+ return {};
+}
+
+std::pair<CVIndexMap &, bool /*already there*/>
+PDBLinker::registerPrecompiledHeaders(uint32_t Signature) {
+ auto Insertion = PrecompTypeIndexMappings.insert({Signature, CVIndexMap()});
+ CVIndexMap &IndexMap = Insertion.first->second;
+ if (!Insertion.second)
+ return {IndexMap, true};
+ // Mark this map as a precompiled types map.
+ IndexMap.IsPrecompiledTypeMap = true;
+ return {IndexMap, false};
+}
+
+Expected<const CVIndexMap &>
+PDBLinker::aquirePrecompObj(ObjFile *File, PrecompRecord Precomp) {
+ // First, check if we already loaded the precompiled headers object with this
+ // signature. Return the type index mapping if we've already seen it.
+ auto R = registerPrecompiledHeaders(Precomp.getSignature());
+ if (R.second)
+ return R.first;
+
+ CVIndexMap &IndexMap = R.first;
+
+ SmallString<128> PrecompPath = Precomp.getPrecompFilePath();
+ sys::fs::make_absolute(PrecompPath);
+
+ // Cross-compile warning: given that Clang doesn't generate LF_PRECOMP
+ // records, we assume the OBJ comes from a Windows build of cl.exe. Thusly,
+ // the paths embedded in the OBJs are in the Windows format.
+ sys::path::native(PrecompPath, sys::path::Style::windows);
+
+ // link.exe requires that a precompiled headers object must always be provided
+ // on the command-line, even if that's not necessary.
+ auto PrecompFilePath = findObjByName(PrecompPath);
+ if (!PrecompFilePath)
+ return errorCodeToError(std::error_code(ENOENT, std::generic_category()));
+
+ ObjFile *CurrentFile = PrecompFilePath->first;
+
+ addObjFile(CurrentFile, &IndexMap);
+
+ if (!CurrentFile->EndPrecomp)
+ fatal(PrecompFilePath->second + " is not a precompiled headers object");
+
+ if (Precomp.getSignature() != CurrentFile->EndPrecomp->getSignature())
+ return make_error<pdb::PDBError>(pdb::pdb_error_code::signature_out_of_date);
+
+ return IndexMap;
+}
+
static bool remapTypeIndex(TypeIndex &TI, ArrayRef<TypeIndex> TypeIndexMap) {
if (TI.isSimple())
return true;
@@ -1027,7 +1242,9 @@ void DebugSHandler::finish() {
File.ModuleDBI->addDebugSubsection(std::move(NewChecksums));
}
-void PDBLinker::addObjFile(ObjFile *File) {
+void PDBLinker::addObjFile(ObjFile *File, CVIndexMap *ExternIndexMap) {
+ if (File->wasProcessedForPDB())
+ return;
// Add a module descriptor for every object file. We need to put an absolute
// path to the object into the PDB. If this is a plain object, we make its
// path absolute. If it's an object in an archive, we make the archive path
@@ -1057,7 +1274,8 @@ void PDBLinker::addObjFile(ObjFile *File
// the PDB first, so that we can get the map from object file type and item
// indices to PDB type and item indices.
CVIndexMap ObjectIndexMap;
- auto IndexMapResult = mergeDebugT(File, ObjectIndexMap);
+ auto IndexMapResult =
+ mergeDebugT(File, ExternIndexMap ? ExternIndexMap : &ObjectIndexMap);
// If the .debug$T sections fail to merge, assume there is no debug info.
if (!IndexMapResult) {
Added: lld/trunk/test/COFF/Inputs/precomp-a.obj
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/Inputs/precomp-a.obj?rev=346154&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lld/trunk/test/COFF/Inputs/precomp-a.obj
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lld/trunk/test/COFF/Inputs/precomp-b.obj
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/Inputs/precomp-b.obj?rev=346154&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lld/trunk/test/COFF/Inputs/precomp-b.obj
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lld/trunk/test/COFF/Inputs/precomp-invalid.obj
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/Inputs/precomp-invalid.obj?rev=346154&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lld/trunk/test/COFF/Inputs/precomp-invalid.obj
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lld/trunk/test/COFF/Inputs/precomp.obj
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/Inputs/precomp.obj?rev=346154&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lld/trunk/test/COFF/Inputs/precomp.obj
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lld/trunk/test/COFF/precomp-link.test
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/precomp-link.test?rev=346154&view=auto
==============================================================================
--- lld/trunk/test/COFF/precomp-link.test (added)
+++ lld/trunk/test/COFF/precomp-link.test Mon Nov 5 11:20:47 2018
@@ -0,0 +1,37 @@
+RUN: lld-link %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj %S/Inputs/precomp.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf
+RUN: llvm-pdbutil dump -types %t.pdb | FileCheck %s
+
+RUN: lld-link %S/Inputs/precomp.obj %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf
+RUN: llvm-pdbutil dump -types %t.pdb | FileCheck %s
+
+RUN: lld-link %S/Inputs/precomp-a.obj %S/Inputs/precomp-invalid.obj %S/Inputs/precomp.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf 2>&1 | FileCheck %s -check-prefix FAILURE
+
+FAILURE: warning: Cannot use debug info for 'precomp-invalid.obj'
+FAILURE-NEXT: failed to load reference '{{.*}}precomp.obj': The signature does not match; the file(s) might be out of date.
+
+CHECK: Types (TPI Stream)
+CHECK-NOT: LF_PRECOMP
+CHECK-NOT: LF_ENDPRECOMP
+
+// precomp.h
+#pragma once
+int Function(char A);
+
+// precomp.cpp
+#include "precomp.h"
+
+// a.cpp
+#include "precomp.h"
+int main(void) {
+ Function('a');
+ return 0;
+}
+
+// b.cpp
+#include "precomp.h"
+int Function(char a) {
+ return (int)a;
+}
+
+// cl.exe precomp.cpp /Z7 /Ycprecomp.h /c
+// cl.exe a.cpp b.cpp /Z7 /Yuprecomp.h /c
More information about the llvm-commits
mailing list