r297655 - Modules: Use hash of PCM content for SIGNATURE
Duncan P. N. Exon Smith via cfe-commits
cfe-commits at lists.llvm.org
Mon Mar 13 11:45:09 PDT 2017
Author: dexonsmith
Date: Mon Mar 13 13:45:08 2017
New Revision: 297655
URL: http://llvm.org/viewvc/llvm-project?rev=297655&view=rev
Log:
Modules: Use hash of PCM content for SIGNATURE
Change ASTFileSignature from a random 32-bit number to the hash of the
PCM content.
- Move definition ASTFileSignature to Basic/Module.h so Module and
ASTSourceDescriptor can use it.
- Change the signature from uint64_t to std::array<uint32_t,5>.
- Stop using (saving/reading) the size and modification time of PCM
files when there is a valid SIGNATURE.
- Add UNHASHED_CONTROL_BLOCK, and use it to store the SIGNATURE record
and other records that shouldn't affect the hash. Because implicit
modules reuses the same file for multiple levels of -Werror, this
includes DIAGNOSTIC_OPTIONS and DIAG_PRAGMA_MAPPINGS.
This helps to solve a PCH + implicit Modules dependency issue: PCH files
are handled by the external build system, whereas implicit modules are
handled by internal compiler build system. This prevents invalidating a
PCH when the compiler overwrites a PCM file with the same content
(modulo the diagnostic differences).
Design and original patch by Manman Ren!
Modified:
cfe/trunk/include/clang/AST/ExternalASTSource.h
cfe/trunk/include/clang/Basic/Module.h
cfe/trunk/include/clang/Driver/CC1Options.td
cfe/trunk/include/clang/Frontend/PCHContainerOperations.h
cfe/trunk/include/clang/Lex/HeaderSearchOptions.h
cfe/trunk/include/clang/Serialization/ASTBitCodes.h
cfe/trunk/include/clang/Serialization/ASTReader.h
cfe/trunk/include/clang/Serialization/ASTWriter.h
cfe/trunk/include/clang/Serialization/Module.h
cfe/trunk/lib/Basic/Module.cpp
cfe/trunk/lib/CodeGen/CGDebugInfo.cpp
cfe/trunk/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
cfe/trunk/lib/Frontend/ASTUnit.cpp
cfe/trunk/lib/Frontend/CompilerInstance.cpp
cfe/trunk/lib/Frontend/CompilerInvocation.cpp
cfe/trunk/lib/Serialization/ASTReader.cpp
cfe/trunk/lib/Serialization/ASTWriter.cpp
cfe/trunk/lib/Serialization/GeneratePCH.cpp
cfe/trunk/lib/Serialization/GlobalModuleIndex.cpp
cfe/trunk/test/Modules/diagnostic-options-out-of-date.m
cfe/trunk/test/Modules/module_file_info.m
cfe/trunk/test/Modules/rebuild.m
Modified: cfe/trunk/include/clang/AST/ExternalASTSource.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/AST/ExternalASTSource.h?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/AST/ExternalASTSource.h (original)
+++ cfe/trunk/include/clang/AST/ExternalASTSource.h Mon Mar 13 13:45:08 2017
@@ -150,20 +150,20 @@ public:
StringRef PCHModuleName;
StringRef Path;
StringRef ASTFile;
- uint64_t Signature = 0;
+ ASTFileSignature Signature;
const Module *ClangModule = nullptr;
public:
ASTSourceDescriptor(){};
ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile,
- uint64_t Signature)
+ ASTFileSignature Signature)
: PCHModuleName(std::move(Name)), Path(std::move(Path)),
ASTFile(std::move(ASTFile)), Signature(Signature){};
ASTSourceDescriptor(const Module &M);
std::string getModuleName() const;
StringRef getPath() const { return Path; }
StringRef getASTFile() const { return ASTFile; }
- uint64_t getSignature() const { return Signature; }
+ ASTFileSignature getSignature() const { return Signature; }
const Module *getModuleOrNull() const { return ClangModule; }
};
Modified: cfe/trunk/include/clang/Basic/Module.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/Module.h?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/Module.h (original)
+++ cfe/trunk/include/clang/Basic/Module.h Mon Mar 13 13:45:08 2017
@@ -42,7 +42,17 @@ class IdentifierInfo;
/// \brief Describes the name of a module.
typedef SmallVector<std::pair<std::string, SourceLocation>, 2> ModuleId;
-
+
+/// The signature of a module, which is a hash of the AST content.
+struct ASTFileSignature : std::array<uint32_t, 5> {
+ ASTFileSignature(std::array<uint32_t, 5> S = {{0}})
+ : std::array<uint32_t, 5>(std::move(S)) {}
+
+ explicit operator bool() const {
+ return *this != std::array<uint32_t, 5>({{0}});
+ }
+};
+
/// \brief Describes a module or submodule.
class Module {
public:
@@ -65,7 +75,7 @@ public:
llvm::PointerUnion<const DirectoryEntry *, const FileEntry *> Umbrella;
/// \brief The module signature.
- uint64_t Signature;
+ ASTFileSignature Signature;
/// \brief The name of the umbrella entry, as written in the module map.
std::string UmbrellaAsWritten;
Modified: cfe/trunk/include/clang/Driver/CC1Options.td
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Driver/CC1Options.td?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/Driver/CC1Options.td (original)
+++ cfe/trunk/include/clang/Driver/CC1Options.td Mon Mar 13 13:45:08 2017
@@ -671,6 +671,8 @@ def nostdsysteminc : Flag<["-"], "nostds
HelpText<"Disable standard system #include directories">;
def fdisable_module_hash : Flag<["-"], "fdisable-module-hash">,
HelpText<"Disable the module hash">;
+def fmodules_hash_content : Flag<["-"], "fmodules-hash-content">,
+ HelpText<"Enable hashing the content of a module file">;
def c_isystem : JoinedOrSeparate<["-"], "c-isystem">, MetaVarName<"<directory>">,
HelpText<"Add directory to the C SYSTEM include search path">;
def objc_isystem : JoinedOrSeparate<["-"], "objc-isystem">,
Modified: cfe/trunk/include/clang/Frontend/PCHContainerOperations.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/PCHContainerOperations.h?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/Frontend/PCHContainerOperations.h (original)
+++ cfe/trunk/include/clang/Frontend/PCHContainerOperations.h Mon Mar 13 13:45:08 2017
@@ -10,6 +10,7 @@
#ifndef LLVM_CLANG_PCH_CONTAINER_OPERATIONS_H
#define LLVM_CLANG_PCH_CONTAINER_OPERATIONS_H
+#include "clang/Basic/Module.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/MemoryBuffer.h"
@@ -29,7 +30,7 @@ class DiagnosticsEngine;
class CompilerInstance;
struct PCHBuffer {
- uint64_t Signature;
+ ASTFileSignature Signature;
llvm::SmallVector<char, 0> Data;
bool IsComplete;
};
Modified: cfe/trunk/include/clang/Lex/HeaderSearchOptions.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/HeaderSearchOptions.h?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/HeaderSearchOptions.h (original)
+++ cfe/trunk/include/clang/Lex/HeaderSearchOptions.h Mon Mar 13 13:45:08 2017
@@ -178,6 +178,8 @@ public:
unsigned ModulesValidateDiagnosticOptions : 1;
+ unsigned ModulesHashContent : 1;
+
HeaderSearchOptions(StringRef _Sysroot = "/")
: Sysroot(_Sysroot), ModuleFormat("raw"), DisableModuleHash(0),
ImplicitModuleMaps(0), ModuleMapFileHomeIsCwd(0),
@@ -186,8 +188,8 @@ public:
UseBuiltinIncludes(true), UseStandardSystemIncludes(true),
UseStandardCXXIncludes(true), UseLibcxx(false), Verbose(false),
ModulesValidateOncePerBuildSession(false),
- ModulesValidateSystemHeaders(false),
- UseDebugInfo(false), ModulesValidateDiagnosticOptions(true) {}
+ ModulesValidateSystemHeaders(false), UseDebugInfo(false),
+ ModulesValidateDiagnosticOptions(true), ModulesHashContent(false) {}
/// AddPath - Add the \p Path path to the specified \p Group list.
void AddPath(StringRef Path, frontend::IncludeDirGroup Group,
Modified: cfe/trunk/include/clang/Serialization/ASTBitCodes.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTBitCodes.h?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/Serialization/ASTBitCodes.h (original)
+++ cfe/trunk/include/clang/Serialization/ASTBitCodes.h Mon Mar 13 13:45:08 2017
@@ -226,7 +226,7 @@ namespace clang {
/// \brief The block containing the detailed preprocessing record.
PREPROCESSOR_DETAIL_BLOCK_ID,
-
+
/// \brief The block containing the submodule structure.
SUBMODULE_BLOCK_ID,
@@ -253,6 +253,12 @@ namespace clang {
/// \brief A block containing a module file extension.
EXTENSION_BLOCK_ID,
+
+ /// A block with unhashed content.
+ ///
+ /// These records should not change the \a ASTFileSignature. See \a
+ /// UnhashedControlBlockRecordTypes for the list of records.
+ UNHASHED_CONTROL_BLOCK_ID,
};
/// \brief Record types that occur within the control block.
@@ -288,9 +294,6 @@ namespace clang {
/// AST file.
MODULE_MAP_FILE,
- /// \brief Record code for the signature that identifiers this AST file.
- SIGNATURE,
-
/// \brief Record code for the module build directory.
MODULE_DIRECTORY,
};
@@ -309,9 +312,6 @@ namespace clang {
/// \brief Record code for the target options table.
TARGET_OPTIONS,
- /// \brief Record code for the diagnostic options table.
- DIAGNOSTIC_OPTIONS,
-
/// \brief Record code for the filesystem options table.
FILE_SYSTEM_OPTIONS,
@@ -322,6 +322,18 @@ namespace clang {
PREPROCESSOR_OPTIONS,
};
+ /// Record codes for the unhashed control block.
+ enum UnhashedControlBlockRecordTypes {
+ /// Record code for the signature that identifiers this AST file.
+ SIGNATURE = 1,
+
+ /// Record code for the diagnostic options table.
+ DIAGNOSTIC_OPTIONS,
+
+ /// Record code for \#pragma diagnostic mappings.
+ DIAG_PRAGMA_MAPPINGS,
+ };
+
/// \brief Record code for extension blocks.
enum ExtensionBlockRecordTypes {
/// Metadata describing this particular extension.
@@ -493,8 +505,7 @@ namespace clang {
// ID 31 used to be a list of offsets to DECL_CXX_BASE_SPECIFIERS records.
- /// \brief Record code for \#pragma diagnostic mappings.
- DIAG_PRAGMA_MAPPINGS = 32,
+ // ID 32 used to be the code for \#pragma diagnostic mappings.
/// \brief Record code for special CUDA declarations.
CUDA_SPECIAL_DECL_REFS = 33,
Modified: cfe/trunk/include/clang/Serialization/ASTReader.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTReader.h?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/Serialization/ASTReader.h (original)
+++ cfe/trunk/include/clang/Serialization/ASTReader.h Mon Mar 13 13:45:08 2017
@@ -1174,7 +1174,7 @@ private:
SourceLocation ImportLoc, ModuleFile *ImportedBy,
SmallVectorImpl<ImportedModule> &Loaded,
off_t ExpectedSize, time_t ExpectedModTime,
- serialization::ASTFileSignature ExpectedSignature,
+ ASTFileSignature ExpectedSignature,
unsigned ClientLoadCapabilities);
ASTReadResult ReadControlBlock(ModuleFile &F,
SmallVectorImpl<ImportedModule> &Loaded,
@@ -1183,7 +1183,22 @@ private:
static ASTReadResult ReadOptionsBlock(
llvm::BitstreamCursor &Stream, unsigned ClientLoadCapabilities,
bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener,
- std::string &SuggestedPredefines, bool ValidateDiagnosticOptions);
+ std::string &SuggestedPredefines);
+
+ /// Read the unhashed control block.
+ ///
+ /// This has no effect on \c F.Stream, instead creating a fresh cursor from
+ /// \c F.Data and reading ahead.
+ ASTReadResult readUnhashedControlBlock(ModuleFile &F, bool WasImportedBy,
+ unsigned ClientLoadCapabilities);
+
+ static ASTReadResult
+ readUnhashedControlBlockImpl(ModuleFile *F, llvm::StringRef StreamData,
+ unsigned ClientLoadCapabilities,
+ bool AllowCompatibleConfigurationMismatch,
+ ASTReaderListener *Listener,
+ bool ValidateDiagnosticOptions);
+
ASTReadResult ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities);
ASTReadResult ReadExtensionBlock(ModuleFile &F);
void ReadModuleOffsetMap(ModuleFile &F) const;
Modified: cfe/trunk/include/clang/Serialization/ASTWriter.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTWriter.h?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/Serialization/ASTWriter.h (original)
+++ cfe/trunk/include/clang/Serialization/ASTWriter.h Mon Mar 13 13:45:08 2017
@@ -106,6 +106,9 @@ private:
/// \brief The bitstream writer used to emit this precompiled header.
llvm::BitstreamWriter &Stream;
+ /// The buffer associated with the bitstream.
+ const SmallVectorImpl<char> &Buffer;
+
/// \brief The ASTContext we're writing.
ASTContext *Context = nullptr;
@@ -425,8 +428,16 @@ private:
void WriteSubStmt(Stmt *S);
void WriteBlockInfoBlock();
- uint64_t WriteControlBlock(Preprocessor &PP, ASTContext &Context,
- StringRef isysroot, const std::string &OutputFile);
+ void WriteControlBlock(Preprocessor &PP, ASTContext &Context,
+ StringRef isysroot, const std::string &OutputFile);
+
+ /// Write out the signature and diagnostic options, and return the signature.
+ ASTFileSignature writeUnhashedControlBlock(Preprocessor &PP,
+ ASTContext &Context);
+
+ /// Calculate hash of the pcm content.
+ static ASTFileSignature createSignature(StringRef Bytes);
+
void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts,
bool Modules);
void WriteSourceManagerBlock(SourceManager &SourceMgr,
@@ -493,14 +504,14 @@ private:
void WriteDeclAbbrevs();
void WriteDecl(ASTContext &Context, Decl *D);
- uint64_t WriteASTCore(Sema &SemaRef,
- StringRef isysroot, const std::string &OutputFile,
- Module *WritingModule);
+ ASTFileSignature WriteASTCore(Sema &SemaRef, StringRef isysroot,
+ const std::string &OutputFile,
+ Module *WritingModule);
public:
/// \brief Create a new precompiled header writer that outputs to
/// the given bitstream.
- ASTWriter(llvm::BitstreamWriter &Stream,
+ ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool IncludeTimestamps = true);
~ASTWriter() override;
@@ -526,9 +537,9 @@ public:
///
/// \return the module signature, which eventually will be a hash of
/// the module but currently is merely a random 32-bit number.
- uint64_t WriteAST(Sema &SemaRef, const std::string &OutputFile,
- Module *WritingModule, StringRef isysroot,
- bool hasErrors = false);
+ ASTFileSignature WriteAST(Sema &SemaRef, const std::string &OutputFile,
+ Module *WritingModule, StringRef isysroot,
+ bool hasErrors = false);
/// \brief Emit a token.
void AddToken(const Token &Tok, RecordDataImpl &Record);
Modified: cfe/trunk/include/clang/Serialization/Module.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/Module.h?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/include/clang/Serialization/Module.h (original)
+++ cfe/trunk/include/clang/Serialization/Module.h Mon Mar 13 13:45:08 2017
@@ -16,6 +16,7 @@
#define LLVM_CLANG_SERIALIZATION_MODULE_H
#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Module.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Serialization/ASTBitCodes.h"
#include "clang/Serialization/ContinuousRangeMap.h"
@@ -89,8 +90,6 @@ public:
bool isNotFound() const { return Val.getInt() == NotFound; }
};
-typedef unsigned ASTFileSignature;
-
/// \brief Information about a module that has been loaded by the ASTReader.
///
/// Each instance of the Module class corresponds to a single AST file, which
@@ -153,9 +152,9 @@ public:
/// \brief The file entry for the module file.
const FileEntry *File = nullptr;
- /// \brief The signature of the module file, which may be used along with size
+ /// The signature of the module file, which may be used instead of the size
/// and modification time to identify this particular file.
- ASTFileSignature Signature = 0;
+ ASTFileSignature Signature;
/// \brief Whether this module has been directly imported by the
/// user.
Modified: cfe/trunk/lib/Basic/Module.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/Module.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/Basic/Module.cpp (original)
+++ cfe/trunk/lib/Basic/Module.cpp Mon Mar 13 13:45:08 2017
@@ -27,7 +27,7 @@ using namespace clang;
Module::Module(StringRef Name, SourceLocation DefinitionLoc, Module *Parent,
bool IsFramework, bool IsExplicit, unsigned VisibilityID)
: Name(Name), DefinitionLoc(DefinitionLoc), Parent(Parent), Directory(),
- Umbrella(), Signature(0), ASTFile(nullptr), VisibilityID(VisibilityID),
+ Umbrella(), ASTFile(nullptr), VisibilityID(VisibilityID),
IsMissingRequirement(false), HasIncompatibleModuleFile(false),
IsAvailable(true), IsFromModuleFile(false), IsFramework(IsFramework),
IsExplicit(IsExplicit), IsSystem(false), IsExternC(false),
Modified: cfe/trunk/lib/CodeGen/CGDebugInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/CodeGen/CGDebugInfo.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/CodeGen/CGDebugInfo.cpp (original)
+++ cfe/trunk/lib/CodeGen/CGDebugInfo.cpp Mon Mar 13 13:45:08 2017
@@ -2053,7 +2053,11 @@ CGDebugInfo::getOrCreateModuleRef(Extern
if (CreateSkeletonCU && IsRootModule) {
// PCH files don't have a signature field in the control block,
// but LLVM detects skeleton CUs by looking for a non-zero DWO id.
- uint64_t Signature = Mod.getSignature() ? Mod.getSignature() : ~1ULL;
+ // We use the lower 64 bits for debug info.
+ uint64_t Signature =
+ Mod.getSignature()
+ ? (uint64_t)Mod.getSignature()[1] << 32 | Mod.getSignature()[0]
+ : ~1ULL;
llvm::DIBuilder DIB(CGM.getModule());
DIB.createCompileUnit(TheCU->getSourceLanguage(),
DIB.createFile(Mod.getModuleName(), Mod.getPath()),
Modified: cfe/trunk/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/CodeGen/ObjectFilePCHContainerOperations.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/CodeGen/ObjectFilePCHContainerOperations.cpp (original)
+++ cfe/trunk/lib/CodeGen/ObjectFilePCHContainerOperations.cpp Mon Mar 13 13:45:08 2017
@@ -171,7 +171,8 @@ public:
// Prepare CGDebugInfo to emit debug info for a clang module.
auto *DI = Builder->getModuleDebugInfo();
StringRef ModuleName = llvm::sys::path::filename(MainFileName);
- DI->setPCHDescriptor({ModuleName, "", OutputFileName, ~1ULL});
+ DI->setPCHDescriptor({ModuleName, "", OutputFileName,
+ ASTFileSignature{{{~0U, ~0U, ~0U, ~0U, ~1U}}}});
DI->setModuleMap(MMap);
}
@@ -241,7 +242,11 @@ public:
// PCH files don't have a signature field in the control block,
// but LLVM detects DWO CUs by looking for a non-zero DWO id.
- uint64_t Signature = Buffer->Signature ? Buffer->Signature : ~1ULL;
+ // We use the lower 64 bits for debug info.
+ uint64_t Signature =
+ Buffer->Signature
+ ? (uint64_t)Buffer->Signature[1] << 32 | Buffer->Signature[0]
+ : ~1ULL;
Builder->getModuleDebugInfo()->setDwoId(Signature);
// Finalize the Builder.
Modified: cfe/trunk/lib/Frontend/ASTUnit.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/ASTUnit.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/ASTUnit.cpp (original)
+++ cfe/trunk/lib/Frontend/ASTUnit.cpp Mon Mar 13 13:45:08 2017
@@ -185,7 +185,7 @@ struct ASTUnit::ASTWriterData {
llvm::BitstreamWriter Stream;
ASTWriter Writer;
- ASTWriterData() : Stream(Buffer), Writer(Stream, { }) { }
+ ASTWriterData() : Stream(Buffer), Writer(Stream, Buffer, {}) {}
};
void ASTUnit::clearFileLevelDecls() {
@@ -2523,7 +2523,7 @@ bool ASTUnit::serialize(raw_ostream &OS)
SmallString<128> Buffer;
llvm::BitstreamWriter Stream(Buffer);
- ASTWriter Writer(Stream, { });
+ ASTWriter Writer(Stream, Buffer, {});
return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS);
}
Modified: cfe/trunk/lib/Frontend/CompilerInstance.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/CompilerInstance.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/CompilerInstance.cpp (original)
+++ cfe/trunk/lib/Frontend/CompilerInstance.cpp Mon Mar 13 13:45:08 2017
@@ -1029,7 +1029,7 @@ static bool compileModuleImpl(CompilerIn
// Remove any macro definitions that are explicitly ignored by the module.
// They aren't supposed to affect how the module is built anyway.
- const HeaderSearchOptions &HSOpts = Invocation->getHeaderSearchOpts();
+ HeaderSearchOptions &HSOpts = Invocation->getHeaderSearchOpts();
PPOpts.Macros.erase(
std::remove_if(PPOpts.Macros.begin(), PPOpts.Macros.end(),
[&HSOpts](const std::pair<std::string, bool> &def) {
@@ -1060,6 +1060,8 @@ static bool compileModuleImpl(CompilerIn
FrontendOpts.DisableFree = false;
FrontendOpts.GenerateGlobalModuleIndex = false;
FrontendOpts.BuildingImplicitModule = true;
+ // Force implicitly-built modules to hash the content of the module file.
+ HSOpts.ModulesHashContent = true;
FrontendOpts.Inputs.clear();
InputKind IK = getSourceInputKindFromOptions(*Invocation->getLangOpts());
Modified: cfe/trunk/lib/Frontend/CompilerInvocation.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/CompilerInvocation.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/CompilerInvocation.cpp (original)
+++ cfe/trunk/lib/Frontend/CompilerInvocation.cpp Mon Mar 13 13:45:08 2017
@@ -1434,6 +1434,7 @@ static void ParseHeaderSearchArgs(Header
for (const Arg *A : Args.filtered(OPT_fprebuilt_module_path))
Opts.AddPrebuiltModulePath(A->getValue());
Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash);
+ Opts.ModulesHashContent = Args.hasArg(OPT_fmodules_hash_content);
Opts.ModulesValidateDiagnosticOptions =
!Args.hasArg(OPT_fmodules_disable_diagnostic_validation);
Opts.ImplicitModuleMaps = Args.hasArg(OPT_fimplicit_module_maps);
Modified: cfe/trunk/lib/Serialization/ASTReader.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTReader.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/Serialization/ASTReader.cpp (original)
+++ cfe/trunk/lib/Serialization/ASTReader.cpp Mon Mar 13 13:45:08 2017
@@ -2163,7 +2163,7 @@ static bool isDiagnosedResult(ASTReader:
ASTReader::ASTReadResult ASTReader::ReadOptionsBlock(
BitstreamCursor &Stream, unsigned ClientLoadCapabilities,
bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener,
- std::string &SuggestedPredefines, bool ValidateDiagnosticOptions) {
+ std::string &SuggestedPredefines) {
if (Stream.EnterSubBlock(OPTIONS_BLOCK_ID))
return Failure;
@@ -2205,15 +2205,6 @@ ASTReader::ASTReadResult ASTReader::Read
break;
}
- case DIAGNOSTIC_OPTIONS: {
- bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0;
- if (ValidateDiagnosticOptions &&
- !AllowCompatibleConfigurationMismatch &&
- ParseDiagnosticOptions(Record, Complain, Listener))
- return OutOfDate;
- break;
- }
-
case FILE_SYSTEM_OPTIONS: {
bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
if (!AllowCompatibleConfigurationMismatch &&
@@ -2254,6 +2245,23 @@ ASTReader::ReadControlBlock(ModuleFile &
return Failure;
}
+ // Lambda to read the unhashed control block the first time it's called.
+ //
+ // For PCM files, the unhashed control block cannot be read until after the
+ // MODULE_NAME record. However, PCH files have no MODULE_NAME, and yet still
+ // need to look ahead before reading the IMPORTS record. For consistency,
+ // this block is always read somehow (see BitstreamEntry::EndBlock).
+ bool HasReadUnhashedControlBlock = false;
+ auto readUnhashedControlBlockOnce = [&]() {
+ if (!HasReadUnhashedControlBlock) {
+ HasReadUnhashedControlBlock = true;
+ if (ASTReadResult Result =
+ readUnhashedControlBlock(F, ImportedBy, ClientLoadCapabilities))
+ return Result;
+ }
+ return Success;
+ };
+
// Read all of the records and blocks in the control block.
RecordData Record;
unsigned NumInputs = 0;
@@ -2266,6 +2274,11 @@ ASTReader::ReadControlBlock(ModuleFile &
Error("malformed block record in AST file");
return Failure;
case llvm::BitstreamEntry::EndBlock: {
+ // Validate the module before returning. This call catches an AST with
+ // no module name and no imports.
+ if (ASTReadResult Result = readUnhashedControlBlockOnce())
+ return Result;
+
// Validate input files.
const HeaderSearchOptions &HSOpts =
PP.getHeaderSearchInfo().getHeaderSearchOpts();
@@ -2337,13 +2350,10 @@ ASTReader::ReadControlBlock(ModuleFile &
// FIXME: Allow this for files explicitly specified with -include-pch.
bool AllowCompatibleConfigurationMismatch =
F.Kind == MK_ExplicitModule || F.Kind == MK_PrebuiltModule;
- const HeaderSearchOptions &HSOpts =
- PP.getHeaderSearchInfo().getHeaderSearchOpts();
Result = ReadOptionsBlock(Stream, ClientLoadCapabilities,
AllowCompatibleConfigurationMismatch,
- *Listener, SuggestedPredefines,
- HSOpts.ModulesValidateDiagnosticOptions);
+ *Listener, SuggestedPredefines);
if (Result == Failure) {
Error("malformed block record in AST file");
return Result;
@@ -2417,12 +2427,13 @@ ASTReader::ReadControlBlock(ModuleFile &
break;
}
- case SIGNATURE:
- assert((!F.Signature || F.Signature == Record[0]) && "signature changed");
- F.Signature = Record[0];
- break;
-
case IMPORTS: {
+ // Validate the AST before processing any imports (otherwise, untangling
+ // them can be error-prone and expensive). A module will have a name and
+ // will already have been validated, but this catches the PCH case.
+ if (ASTReadResult Result = readUnhashedControlBlockOnce())
+ return Result;
+
// Load each of the imported PCH files.
unsigned Idx = 0, N = Record.size();
while (Idx < N) {
@@ -2435,7 +2446,10 @@ ASTReader::ReadControlBlock(ModuleFile &
ReadUntranslatedSourceLocation(Record[Idx++]);
off_t StoredSize = (off_t)Record[Idx++];
time_t StoredModTime = (time_t)Record[Idx++];
- ASTFileSignature StoredSignature = Record[Idx++];
+ ASTFileSignature StoredSignature = {
+ {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
+ (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
+ (uint32_t)Record[Idx++]}}};
auto ImportedFile = ReadPath(F, Record, Idx);
// If our client can't cope with us being out of date, we can't cope with
@@ -2487,6 +2501,12 @@ ASTReader::ReadControlBlock(ModuleFile &
F.ModuleName = Blob;
if (Listener)
Listener->ReadModuleName(F.ModuleName);
+
+ // Validate the AST as soon as we have a name so we can exit early on
+ // failure.
+ if (ASTReadResult Result = readUnhashedControlBlockOnce())
+ return Result;
+
break;
case MODULE_DIRECTORY: {
@@ -3076,14 +3096,6 @@ ASTReader::ReadASTBlock(ModuleFile &F, u
F.ObjCCategories.swap(Record);
break;
- case DIAG_PRAGMA_MAPPINGS:
- if (F.PragmaDiagMappings.empty())
- F.PragmaDiagMappings.swap(Record);
- else
- F.PragmaDiagMappings.insert(F.PragmaDiagMappings.end(),
- Record.begin(), Record.end());
- break;
-
case CUDA_SPECIAL_DECL_REFS:
// Later tables overwrite earlier ones.
// FIXME: Modules will have trouble with this.
@@ -3657,10 +3669,10 @@ ASTReader::ASTReadResult ASTReader::Read
unsigned NumModules = ModuleMgr.size();
SmallVector<ImportedModule, 4> Loaded;
- switch(ASTReadResult ReadResult = ReadASTCore(FileName, Type, ImportLoc,
- /*ImportedBy=*/nullptr, Loaded,
- 0, 0, 0,
- ClientLoadCapabilities)) {
+ switch (ASTReadResult ReadResult =
+ ReadASTCore(FileName, Type, ImportLoc,
+ /*ImportedBy=*/nullptr, Loaded, 0, 0,
+ ASTFileSignature(), ClientLoadCapabilities)) {
case Failure:
case Missing:
case OutOfDate:
@@ -4021,6 +4033,12 @@ ASTReader::ReadASTCore(StringRef FileNam
Loaded.push_back(ImportedModule(M, ImportedBy, ImportLoc));
return Success;
+ case UNHASHED_CONTROL_BLOCK_ID:
+ // This block is handled using look-ahead during ReadControlBlock. We
+ // shouldn't get here!
+ Error("malformed block record in AST file");
+ return Failure;
+
default:
if (Stream.SkipBlock()) {
Error("malformed block record in AST file");
@@ -4033,6 +4051,93 @@ ASTReader::ReadASTCore(StringRef FileNam
return Success;
}
+ASTReader::ASTReadResult
+ASTReader::readUnhashedControlBlock(ModuleFile &F, bool WasImportedBy,
+ unsigned ClientLoadCapabilities) {
+ const HeaderSearchOptions &HSOpts =
+ PP.getHeaderSearchInfo().getHeaderSearchOpts();
+ bool AllowCompatibleConfigurationMismatch =
+ F.Kind == MK_ExplicitModule || F.Kind == MK_PrebuiltModule;
+
+ ASTReadResult Result = readUnhashedControlBlockImpl(
+ &F, F.Data, ClientLoadCapabilities, AllowCompatibleConfigurationMismatch,
+ Listener.get(),
+ WasImportedBy ? false : HSOpts.ModulesValidateDiagnosticOptions);
+
+ if (DisableValidation || WasImportedBy ||
+ (AllowConfigurationMismatch && Result == ConfigurationMismatch))
+ return Success;
+
+ if (Result == Failure)
+ Error("malformed block record in AST file");
+
+ return Result;
+}
+
+ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
+ ModuleFile *F, llvm::StringRef StreamData, unsigned ClientLoadCapabilities,
+ bool AllowCompatibleConfigurationMismatch, ASTReaderListener *Listener,
+ bool ValidateDiagnosticOptions) {
+ // Initialize a stream.
+ BitstreamCursor Stream(StreamData);
+
+ // Sniff for the signature.
+ if (!startsWithASTFileMagic(Stream))
+ return Failure;
+
+ // Scan for the UNHASHED_CONTROL_BLOCK_ID block.
+ if (SkipCursorToBlock(Stream, UNHASHED_CONTROL_BLOCK_ID))
+ return Failure;
+
+ // Read all of the records in the options block.
+ RecordData Record;
+ ASTReadResult Result = Success;
+ while (1) {
+ llvm::BitstreamEntry Entry = Stream.advance();
+
+ switch (Entry.Kind) {
+ case llvm::BitstreamEntry::Error:
+ case llvm::BitstreamEntry::SubBlock:
+ return Failure;
+
+ case llvm::BitstreamEntry::EndBlock:
+ return Result;
+
+ case llvm::BitstreamEntry::Record:
+ // The interesting case.
+ break;
+ }
+
+ // Read and process a record.
+ Record.clear();
+ switch (
+ (UnhashedControlBlockRecordTypes)Stream.readRecord(Entry.ID, Record)) {
+ case SIGNATURE: {
+ if (F)
+ std::copy(Record.begin(), Record.end(), F->Signature.data());
+ break;
+ }
+ case DIAGNOSTIC_OPTIONS: {
+ bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0;
+ if (Listener && ValidateDiagnosticOptions &&
+ !AllowCompatibleConfigurationMismatch &&
+ ParseDiagnosticOptions(Record, Complain, *Listener))
+ return OutOfDate;
+ break;
+ }
+ case DIAG_PRAGMA_MAPPINGS:
+ if (!F)
+ break;
+ if (F->PragmaDiagMappings.empty())
+ F->PragmaDiagMappings.swap(Record);
+ else
+ F->PragmaDiagMappings.insert(F->PragmaDiagMappings.end(),
+ Record.begin(), Record.end());
+ break;
+ }
+ }
+}
+
/// Parse a record and blob containing module file extension metadata.
static bool parseModuleFileExtensionMetadata(
const SmallVectorImpl<uint64_t> &Record,
@@ -4249,23 +4354,24 @@ void ASTReader::finalizeForWriting() {
static ASTFileSignature readASTFileSignature(StringRef PCH) {
BitstreamCursor Stream(PCH);
if (!startsWithASTFileMagic(Stream))
- return 0;
+ return ASTFileSignature();
- // Scan for the CONTROL_BLOCK_ID block.
- if (SkipCursorToBlock(Stream, CONTROL_BLOCK_ID))
- return 0;
+ // Scan for the UNHASHED_CONTROL_BLOCK_ID block.
+ if (SkipCursorToBlock(Stream, UNHASHED_CONTROL_BLOCK_ID))
+ return ASTFileSignature();
- // Scan for SIGNATURE inside the control block.
+ // Scan for SIGNATURE inside the diagnostic options block.
ASTReader::RecordData Record;
while (true) {
llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
if (Entry.Kind != llvm::BitstreamEntry::Record)
- return 0;
+ return ASTFileSignature();
Record.clear();
StringRef Blob;
if (SIGNATURE == Stream.readRecord(Entry.ID, Record, &Blob))
- return Record[0];
+ return {{{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
+ (uint32_t)Record[3], (uint32_t)Record[4]}}};
}
}
@@ -4384,7 +4490,8 @@ bool ASTReader::readASTFileControlBlock(
}
// Initialize the stream
- BitstreamCursor Stream(PCHContainerRdr.ExtractPCH(**Buffer));
+ StringRef Bytes = PCHContainerRdr.ExtractPCH(**Buffer);
+ BitstreamCursor Stream(Bytes);
// Sniff for the signature.
if (!startsWithASTFileMagic(Stream))
@@ -4412,8 +4519,7 @@ bool ASTReader::readASTFileControlBlock(
std::string IgnoredSuggestedPredefines;
if (ReadOptionsBlock(Stream, ARR_ConfigurationMismatch | ARR_OutOfDate,
/*AllowCompatibleConfigurationMismatch*/ false,
- Listener, IgnoredSuggestedPredefines,
- ValidateDiagnosticOptions) != Success)
+ Listener, IgnoredSuggestedPredefines) != Success)
return true;
break;
}
@@ -4534,6 +4640,7 @@ bool ASTReader::readASTFileControlBlock(
// Look for module file extension blocks, if requested.
if (FindModuleFileExtensions) {
+ BitstreamCursor SavedStream = Stream;
while (!SkipCursorToBlock(Stream, EXTENSION_BLOCK_ID)) {
bool DoneWithExtensionBlock = false;
while (!DoneWithExtensionBlock) {
@@ -4572,8 +4679,16 @@ bool ASTReader::readASTFileControlBlock(
}
}
}
+ Stream = SavedStream;
}
+ // Scan for the UNHASHED_CONTROL_BLOCK_ID block.
+ if (readUnhashedControlBlockImpl(
+ nullptr, Bytes, ARR_ConfigurationMismatch | ARR_OutOfDate,
+ /*AllowCompatibleConfigurationMismatch*/ false, &Listener,
+ ValidateDiagnosticOptions) != Success)
+ return true;
+
return false;
}
Modified: cfe/trunk/lib/Serialization/ASTWriter.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTWriter.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/Serialization/ASTWriter.cpp (original)
+++ cfe/trunk/lib/Serialization/ASTWriter.cpp Mon Mar 13 13:45:08 2017
@@ -18,8 +18,8 @@
#include "clang/AST/ASTContext.h"
#include "clang/AST/ASTUnresolvedSet.h"
#include "clang/AST/Decl.h"
-#include "clang/AST/DeclContextInternals.h"
#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclContextInternals.h"
#include "clang/AST/DeclFriend.h"
#include "clang/AST/DeclTemplate.h"
#include "clang/AST/Expr.h"
@@ -33,8 +33,8 @@
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/FileSystemOptions.h"
-#include "clang/Basic/LangOptions.h"
#include "clang/Basic/LLVM.h"
+#include "clang/Basic/LangOptions.h"
#include "clang/Basic/Module.h"
#include "clang/Basic/ObjCRuntime.h"
#include "clang/Basic/SourceManager.h"
@@ -64,9 +64,9 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Bitcode/BitCodes.h"
#include "llvm/Bitcode/BitstreamWriter.h"
@@ -79,6 +79,7 @@
#include "llvm/Support/OnDiskHashTable.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
+#include "llvm/Support/SHA1.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
@@ -1016,7 +1017,6 @@ void ASTWriter::WriteBlockInfoBlock() {
// Control Block.
BLOCK(CONTROL_BLOCK);
RECORD(METADATA);
- RECORD(SIGNATURE);
RECORD(MODULE_NAME);
RECORD(MODULE_DIRECTORY);
RECORD(MODULE_MAP_FILE);
@@ -1029,7 +1029,6 @@ void ASTWriter::WriteBlockInfoBlock() {
BLOCK(OPTIONS_BLOCK);
RECORD(LANGUAGE_OPTIONS);
RECORD(TARGET_OPTIONS);
- RECORD(DIAGNOSTIC_OPTIONS);
RECORD(FILE_SYSTEM_OPTIONS);
RECORD(HEADER_SEARCH_OPTIONS);
RECORD(PREPROCESSOR_OPTIONS);
@@ -1065,7 +1064,6 @@ void ASTWriter::WriteBlockInfoBlock() {
RECORD(UPDATE_VISIBLE);
RECORD(DECL_UPDATE_OFFSETS);
RECORD(DECL_UPDATES);
- RECORD(DIAG_PRAGMA_MAPPINGS);
RECORD(CUDA_SPECIAL_DECL_REFS);
RECORD(HEADER_SEARCH_TABLE);
RECORD(FP_PRAGMA_OPTIONS);
@@ -1258,6 +1256,11 @@ void ASTWriter::WriteBlockInfoBlock() {
BLOCK(EXTENSION_BLOCK);
RECORD(EXTENSION_METADATA);
+ BLOCK(UNHASHED_CONTROL_BLOCK);
+ RECORD(SIGNATURE);
+ RECORD(DIAGNOSTIC_OPTIONS);
+ RECORD(DIAG_PRAGMA_MAPPINGS);
+
#undef RECORD
#undef BLOCK
Stream.ExitBlock();
@@ -1320,21 +1323,73 @@ adjustFilenameForRelocatableAST(const ch
return Filename + Pos;
}
-static ASTFileSignature getSignature() {
- while (true) {
- if (ASTFileSignature S = llvm::sys::Process::GetRandomNumber())
- return S;
- // Rely on GetRandomNumber to eventually return non-zero...
+ASTFileSignature ASTWriter::createSignature(StringRef Bytes) {
+ // Calculate the hash till start of UNHASHED_CONTROL_BLOCK.
+ llvm::SHA1 Hasher;
+ Hasher.update(ArrayRef<uint8_t>(Bytes.bytes_begin(), Bytes.size()));
+ auto Hash = Hasher.result();
+
+ // Convert to an array [5*i32].
+ ASTFileSignature Signature;
+ auto LShift = [&](unsigned char Val, unsigned Shift) {
+ return (uint32_t)Val << Shift;
+ };
+ for (int I = 0; I != 5; ++I)
+ Signature[I] = LShift(Hash[I * 4 + 0], 24) | LShift(Hash[I * 4 + 1], 16) |
+ LShift(Hash[I * 4 + 2], 8) | LShift(Hash[I * 4 + 3], 0);
+
+ return Signature;
+}
+
+ASTFileSignature ASTWriter::writeUnhashedControlBlock(Preprocessor &PP,
+ ASTContext &Context) {
+ // Flush first to prepare the PCM hash (signature).
+ Stream.FlushToWord();
+ auto StartOfUnhashedControl = Stream.GetCurrentBitNo() >> 3;
+
+ // Enter the block and prepare to write records.
+ RecordData Record;
+ Stream.EnterSubblock(UNHASHED_CONTROL_BLOCK_ID, 5);
+
+ // For implicit modules, write the hash of the PCM as its signature.
+ ASTFileSignature Signature;
+ if (WritingModule &&
+ PP.getHeaderSearchInfo().getHeaderSearchOpts().ModulesHashContent) {
+ Signature = createSignature(StringRef(Buffer.begin(), StartOfUnhashedControl));
+ Record.append(Signature.begin(), Signature.end());
+ Stream.EmitRecord(SIGNATURE, Record);
+ Record.clear();
}
+
+ // Diagnostic options.
+ const auto &Diags = Context.getDiagnostics();
+ const DiagnosticOptions &DiagOpts = Diags.getDiagnosticOptions();
+#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name);
+#define ENUM_DIAGOPT(Name, Type, Bits, Default) \
+ Record.push_back(static_cast<unsigned>(DiagOpts.get##Name()));
+#include "clang/Basic/DiagnosticOptions.def"
+ Record.push_back(DiagOpts.Warnings.size());
+ for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I)
+ AddString(DiagOpts.Warnings[I], Record);
+ Record.push_back(DiagOpts.Remarks.size());
+ for (unsigned I = 0, N = DiagOpts.Remarks.size(); I != N; ++I)
+ AddString(DiagOpts.Remarks[I], Record);
+ // Note: we don't serialize the log or serialization file names, because they
+ // are generally transient files and will almost always be overridden.
+ Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record);
+
+ // Write out the diagnostic/pragma mappings.
+ WritePragmaDiagnosticMappings(Diags, /* IsModule = */ WritingModule);
+
+ // Leave the options block.
+ Stream.ExitBlock();
+ return Signature;
}
/// \brief Write the control block.
-uint64_t ASTWriter::WriteControlBlock(Preprocessor &PP,
- ASTContext &Context,
- StringRef isysroot,
- const std::string &OutputFile) {
- ASTFileSignature Signature = 0;
-
+void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
+ StringRef isysroot,
+ const std::string &OutputFile) {
using namespace llvm;
Stream.EnterSubblock(CONTROL_BLOCK_ID, 5);
RecordData Record;
@@ -1362,15 +1417,6 @@ uint64_t ASTWriter::WriteControlBlock(Pr
getClangFullRepositoryVersion());
}
if (WritingModule) {
- // For implicit modules we output a signature that we can use to ensure
- // duplicate module builds don't collide in the cache as their output order
- // is non-deterministic.
- // FIXME: Remove this when output is deterministic.
- if (Context.getLangOpts().ImplicitModules) {
- Signature = getSignature();
- RecordData::value_type Record[] = {Signature};
- Stream.EmitRecord(SIGNATURE, Record);
- }
// Module name
auto Abbrev = std::make_shared<BitCodeAbbrev>();
@@ -1443,9 +1489,15 @@ uint64_t ASTWriter::WriteControlBlock(Pr
Record.push_back((unsigned)M.Kind); // FIXME: Stable encoding
AddSourceLocation(M.ImportLoc, Record);
- Record.push_back(M.File->getSize());
- Record.push_back(getTimestampForOutput(M.File));
- Record.push_back(M.Signature);
+
+ // If we have calculated signature, there is no need to store
+ // the size or timestamp.
+ Record.push_back(M.Signature ? 0 : M.File->getSize());
+ Record.push_back(M.Signature ? 0 : getTimestampForOutput(M.File));
+
+ for (auto I : M.Signature)
+ Record.push_back(I);
+
AddPath(M.FileName, Record);
}
Stream.EmitRecord(IMPORTS, Record);
@@ -1508,24 +1560,6 @@ uint64_t ASTWriter::WriteControlBlock(Pr
}
Stream.EmitRecord(TARGET_OPTIONS, Record);
- // Diagnostic options.
- Record.clear();
- const DiagnosticOptions &DiagOpts
- = Context.getDiagnostics().getDiagnosticOptions();
-#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name);
-#define ENUM_DIAGOPT(Name, Type, Bits, Default) \
- Record.push_back(static_cast<unsigned>(DiagOpts.get##Name()));
-#include "clang/Basic/DiagnosticOptions.def"
- Record.push_back(DiagOpts.Warnings.size());
- for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I)
- AddString(DiagOpts.Warnings[I], Record);
- Record.push_back(DiagOpts.Remarks.size());
- for (unsigned I = 0, N = DiagOpts.Remarks.size(); I != N; ++I)
- AddString(DiagOpts.Remarks[I], Record);
- // Note: we don't serialize the log or serialization file names, because they
- // are generally transient files and will almost always be overridden.
- Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record);
-
// File system options.
Record.clear();
const FileSystemOptions &FSOpts =
@@ -1639,7 +1673,6 @@ uint64_t ASTWriter::WriteControlBlock(Pr
PP.getHeaderSearchInfo().getHeaderSearchOpts(),
PP.getLangOpts().Modules);
Stream.ExitBlock();
- return Signature;
}
namespace {
@@ -4267,9 +4300,10 @@ void ASTWriter::SetSelectorOffset(Select
}
ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream,
+ SmallVectorImpl<char> &Buffer,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool IncludeTimestamps)
- : Stream(Stream), IncludeTimestamps(IncludeTimestamps) {
+ : Stream(Stream), Buffer(Buffer), IncludeTimestamps(IncludeTimestamps) {
for (const auto &Ext : Extensions) {
if (auto Writer = Ext->createExtensionWriter(*this))
ModuleFileExtensionWriters.push_back(std::move(Writer));
@@ -4289,9 +4323,10 @@ time_t ASTWriter::getTimestampForOutput(
return IncludeTimestamps ? E->getModificationTime() : 0;
}
-uint64_t ASTWriter::WriteAST(Sema &SemaRef, const std::string &OutputFile,
- Module *WritingModule, StringRef isysroot,
- bool hasErrors) {
+ASTFileSignature ASTWriter::WriteAST(Sema &SemaRef,
+ const std::string &OutputFile,
+ Module *WritingModule, StringRef isysroot,
+ bool hasErrors) {
WritingAST = true;
ASTHasCompilerErrors = hasErrors;
@@ -4327,9 +4362,9 @@ static void AddLazyVectorDecls(ASTWriter
}
}
-uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
- const std::string &OutputFile,
- Module *WritingModule) {
+ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
+ const std::string &OutputFile,
+ Module *WritingModule) {
using namespace llvm;
bool isModule = WritingModule != nullptr;
@@ -4477,7 +4512,7 @@ uint64_t ASTWriter::WriteASTCore(Sema &S
}
// Write the control block
- uint64_t Signature = WriteControlBlock(PP, Context, isysroot, OutputFile);
+ WriteControlBlock(PP, Context, isysroot, OutputFile);
// Write the remaining AST contents.
Stream.EnterSubblock(AST_BLOCK_ID, 5);
@@ -4694,7 +4729,6 @@ uint64_t ASTWriter::WriteASTCore(Sema &S
WriteOpenCLExtensionTypes(SemaRef);
WriteOpenCLExtensionDecls(SemaRef);
WriteCUDAPragmas(SemaRef);
- WritePragmaDiagnosticMappings(Context.getDiagnostics(), isModule);
// If we're emitting a module, write out the submodule information.
if (WritingModule)
@@ -4823,7 +4857,7 @@ uint64_t ASTWriter::WriteASTCore(Sema &S
for (const auto &ExtWriter : ModuleFileExtensionWriters)
WriteModuleFileExtension(SemaRef, *ExtWriter);
- return Signature;
+ return writeUnhashedControlBlock(PP, Context);
}
void ASTWriter::WriteDeclUpdatesBlocks(RecordDataImpl &OffsetsRecord) {
Modified: cfe/trunk/lib/Serialization/GeneratePCH.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/GeneratePCH.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/Serialization/GeneratePCH.cpp (original)
+++ cfe/trunk/lib/Serialization/GeneratePCH.cpp Mon Mar 13 13:45:08 2017
@@ -28,7 +28,7 @@ PCHGenerator::PCHGenerator(
bool AllowASTWithErrors, bool IncludeTimestamps)
: PP(PP), OutputFile(OutputFile), isysroot(isysroot.str()),
SemaPtr(nullptr), Buffer(Buffer), Stream(Buffer->Data),
- Writer(Stream, Extensions, IncludeTimestamps),
+ Writer(Stream, Buffer->Data, Extensions, IncludeTimestamps),
AllowASTWithErrors(AllowASTWithErrors) {
Buffer->IsComplete = false;
}
Modified: cfe/trunk/lib/Serialization/GlobalModuleIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/GlobalModuleIndex.cpp?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/lib/Serialization/GlobalModuleIndex.cpp (original)
+++ cfe/trunk/lib/Serialization/GlobalModuleIndex.cpp Mon Mar 13 13:45:08 2017
@@ -376,6 +376,15 @@ namespace {
/// \brief The set of modules on which this module depends. Each entry is
/// a module ID.
SmallVector<unsigned, 4> Dependencies;
+ ASTFileSignature Signature;
+ };
+
+ struct ImportedModuleFileInfo {
+ off_t StoredSize;
+ time_t StoredModTime;
+ ASTFileSignature StoredSignature;
+ ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
+ : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
};
/// \brief Builder that generates the global module index file.
@@ -383,12 +392,20 @@ namespace {
FileManager &FileMgr;
const PCHContainerReader &PCHContainerRdr;
- /// \brief Mapping from files to module file information.
+ /// Mapping from files to module file information.
typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
- /// \brief Information about each of the known module files.
+ /// Information about each of the known module files.
ModuleFilesMap ModuleFiles;
+ /// \brief Mapping from the imported module file to the imported
+ /// information.
+ typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
+ ImportedModuleFilesMap;
+
+ /// \brief Information about each importing of a module file.
+ ImportedModuleFilesMap ImportedModuleFiles;
+
/// \brief Mapping from identifiers to the list of module file IDs that
/// consider this identifier to be interesting.
typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
@@ -424,7 +441,8 @@ namespace {
bool loadModuleFile(const FileEntry *File);
/// \brief Write the index to the given bitstream.
- void writeIndex(llvm::BitstreamWriter &Stream);
+ /// \returns true if an error occurred, false otherwise.
+ bool writeIndex(llvm::BitstreamWriter &Stream);
};
}
@@ -515,7 +533,7 @@ bool GlobalModuleIndexBuilder::loadModul
unsigned ID = getModuleFileInfo(File).ID;
// Search for the blocks and records we care about.
- enum { Other, ControlBlock, ASTBlock } State = Other;
+ enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
bool Done = false;
while (!Done) {
llvm::BitstreamEntry Entry = InStream.advance();
@@ -553,6 +571,15 @@ bool GlobalModuleIndexBuilder::loadModul
continue;
}
+ if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
+ if (InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
+ return true;
+
+ // Found the Diagnostic Options block.
+ State = DiagnosticOptionsBlock;
+ continue;
+ }
+
if (InStream.SkipBlock())
return true;
@@ -587,7 +614,10 @@ bool GlobalModuleIndexBuilder::loadModul
// Skip the stored signature.
// FIXME: we could read the signature out of the import and validate it.
- Idx++;
+ ASTFileSignature StoredSignature = {
+ {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
+ (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
+ (uint32_t)Record[Idx++]}}};
// Retrieve the imported file name.
unsigned Length = Record[Idx++];
@@ -599,11 +629,16 @@ bool GlobalModuleIndexBuilder::loadModul
const FileEntry *DependsOnFile
= FileMgr.getFile(ImportedFile, /*openFile=*/false,
/*cacheFailure=*/false);
- if (!DependsOnFile ||
- (StoredSize != DependsOnFile->getSize()) ||
- (StoredModTime != DependsOnFile->getModificationTime()))
+
+ if (!DependsOnFile)
return true;
+ // Save the information in ImportedModuleFileInfo so we can verify after
+ // loading all pcms.
+ ImportedModuleFiles.insert(std::make_pair(
+ DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
+ StoredSignature)));
+
// Record the dependency.
unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
@@ -632,6 +667,12 @@ bool GlobalModuleIndexBuilder::loadModul
}
}
+ // Get Signature.
+ if (State == DiagnosticOptionsBlock && Code == SIGNATURE)
+ getModuleFileInfo(File).Signature = {
+ {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
+ (uint32_t)Record[3], (uint32_t)Record[4]}}};
+
// We don't care about this record.
}
@@ -680,7 +721,20 @@ public:
}
-void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
+bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
+ for (auto MapEntry : ImportedModuleFiles) {
+ auto *File = MapEntry.first;
+ ImportedModuleFileInfo &Info = MapEntry.second;
+ if (getModuleFileInfo(File).Signature) {
+ if (getModuleFileInfo(File).Signature != Info.StoredSignature)
+ // Verify Signature.
+ return true;
+ } else if (Info.StoredSize != File->getSize() ||
+ Info.StoredModTime != File->getModificationTime())
+ // Verify Size and ModTime.
+ return true;
+ }
+
using namespace llvm;
// Emit the file header.
@@ -756,6 +810,7 @@ void GlobalModuleIndexBuilder::writeInde
}
Stream.ExitBlock();
+ return false;
}
GlobalModuleIndex::ErrorCode
@@ -816,7 +871,8 @@ GlobalModuleIndex::writeIndex(FileManage
SmallVector<char, 16> OutputBuffer;
{
llvm::BitstreamWriter OutputStream(OutputBuffer);
- Builder.writeIndex(OutputStream);
+ if (Builder.writeIndex(OutputStream))
+ return EC_IOError;
}
// Write the global index file to a temporary file.
Modified: cfe/trunk/test/Modules/diagnostic-options-out-of-date.m
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/diagnostic-options-out-of-date.m?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/test/Modules/diagnostic-options-out-of-date.m (original)
+++ cfe/trunk/test/Modules/diagnostic-options-out-of-date.m Mon Mar 13 13:45:08 2017
@@ -7,6 +7,16 @@
// RUN: %clang_cc1 -Werror -Wconversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s -fmodules-disable-diagnostic-validation
// Make sure we don't error out when using the pch
// RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -fsyntax-only -I %S/Inputs -include-pch %t.pch %s -verify -fmodules-disable-diagnostic-validation
+
+// Build A.pcm
+// RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s
+// Build pch that imports A.pcm
+// RUN: %clang_cc1 -Werror -Wno-conversion -emit-pch -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -o %t.pch -I %S/Inputs -x objective-c-header %S/Inputs/pch-import-module-out-of-date.pch
+// We will rebuild A.pcm and overwrite the original A.pcm that the pch imports, but the two versions have the same hash.
+// RUN: %clang_cc1 -Werror -Wconversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s
+// Make sure we don't error out when using the pch
+// RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -fsyntax-only -I %S/Inputs -include-pch %t.pch %s -verify
+
// expected-no-diagnostics
@import DiagOutOfDate;
Modified: cfe/trunk/test/Modules/module_file_info.m
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/module_file_info.m?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/test/Modules/module_file_info.m (original)
+++ cfe/trunk/test/Modules/module_file_info.m Mon Mar 13 13:45:08 2017
@@ -29,11 +29,6 @@
// CHECK: CPU:
// CHECK: ABI:
-// CHECK: Diagnostic options:
-// CHECK: IgnoreWarnings: Yes
-// CHECK: Diagnostic flags:
-// CHECK: -Wunused
-
// CHECK: Header search options:
// CHECK: System root [-isysroot=]: '/'
// CHECK: Resource dir [ -resource-dir=]: '{{.*}}clang{{.*}}'
@@ -48,3 +43,8 @@
// CHECK: Predefined macros:
// CHECK: -DBLARG
// CHECK: -DWIBBLE=WOBBLE
+
+// CHECK: Diagnostic options:
+// CHECK: IgnoreWarnings: Yes
+// CHECK: Diagnostic flags:
+// CHECK: -Wunused
Modified: cfe/trunk/test/Modules/rebuild.m
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/rebuild.m?rev=297655&r1=297654&r2=297655&view=diff
==============================================================================
--- cfe/trunk/test/Modules/rebuild.m (original)
+++ cfe/trunk/test/Modules/rebuild.m Mon Mar 13 13:45:08 2017
@@ -19,11 +19,10 @@
// RUN: diff %t/Module.size %t/Module.size.saved
// RUN: cp %t/Module.pcm %t/Module.pcm.saved.2
-// But the signature at least is expected to change, so we rebuild DependsOnModule.
-// NOTE: if we change how the signature is created, this test may need updating.
+// The signature is the hash of the PCM content, we will not rebuild rebuild DependsOnModule.
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -fsyntax-only -F %S/Inputs %s
// RUN: diff %t/Module.pcm %t/Module.pcm.saved.2
-// RUN: not diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved
+// RUN: diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved
// Rebuild Module, reset its timestamp, and verify its size hasn't changed
// RUN: rm %t/Module.pcm
@@ -34,10 +33,9 @@
// RUN: cp %t/Module.pcm %t/Module.pcm.saved.2
// Verify again with Module pre-imported.
-// NOTE: if we change how the signature is created, this test may need updating.
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -fsyntax-only -F %S/Inputs %s
// RUN: diff %t/Module.pcm %t/Module.pcm.saved.2
-// RUN: not diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved
+// RUN: diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved
#ifdef PREIMPORT
@import Module;
More information about the cfe-commits
mailing list