[clang] 40472ef - [clang][modules] Serialize VFS overlay paths into PCMs
Jan Svoboda via cfe-commits
cfe-commits at lists.llvm.org
Fri Dec 2 16:12:22 PST 2022
Author: Jan Svoboda
Date: 2022-12-02T16:12:16-08:00
New Revision: 40472ef14cd3bbed665789825b47d055e0a83402
URL: https://github.com/llvm/llvm-project/commit/40472ef14cd3bbed665789825b47d055e0a83402
DIFF: https://github.com/llvm/llvm-project/commit/40472ef14cd3bbed665789825b47d055e0a83402.diff
LOG: [clang][modules] Serialize VFS overlay paths into PCMs
With implicitly built modules, the importing `CompilerInstance` assumes PCMs were built in a "compatible way" (i.e. with similarly set up instance). Either because their context hash matches, or because this instance has just built them.
There are some use-cases, however, where this assumption doesn't hold, libclang/c-index-test being one of them. There, the importing instance (or `ASTUnit`) is being set up while the PCM file is being deserialized. Until now, we've assumed the serialized paths to input files are the actual on-disk files, meaning the default physical VFS was always able to resolve them. This won't be the case after D135636. Therefore, this patch makes sure `ASTUnit` is initialized with the same VFS as the PCM it's deserializing - by storing paths to the VFS overlay files into the PCM itself.
For the VFS overlay files to be adopted at the very start of PCM deserialization, they are stored in a new section in the unhashed control block, together with header search paths and system header prefixes. The move to the unhashed control block should be safe: if two modules were built with different header search paths and they produced different results, the hashed part of the PCM file will reflect that.
Reviewed By: akyrtzi, benlangmuir
Differential Revision: https://reviews.llvm.org/D135634
Added:
Modified:
clang/include/clang/Frontend/CompilerInvocation.h
clang/include/clang/Serialization/ASTBitCodes.h
clang/include/clang/Serialization/ASTReader.h
clang/lib/Frontend/ASTUnit.cpp
clang/lib/Frontend/CompilerInvocation.cpp
clang/lib/Serialization/ASTReader.cpp
clang/lib/Serialization/ASTWriter.cpp
Removed:
################################################################################
diff --git a/clang/include/clang/Frontend/CompilerInvocation.h b/clang/include/clang/Frontend/CompilerInvocation.h
index 9cf28c52f4d9..254f048ed3c7 100644
--- a/clang/include/clang/Frontend/CompilerInvocation.h
+++ b/clang/include/clang/Frontend/CompilerInvocation.h
@@ -294,6 +294,11 @@ IntrusiveRefCntPtr<llvm::vfs::FileSystem> createVFSFromCompilerInvocation(
const CompilerInvocation &CI, DiagnosticsEngine &Diags,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS);
+IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+createVFSFromOverlayFiles(ArrayRef<std::string> VFSOverlayFiles,
+ DiagnosticsEngine &Diags,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS);
+
} // namespace clang
#endif // LLVM_CLANG_FRONTEND_COMPILERINVOCATION_H
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index ceaade4a6e1e..3f6b22a9a548 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -41,7 +41,7 @@ namespace serialization {
/// Version 4 of AST files also requires that the version control branch and
/// revision match exactly, since there is no backward compatibility of
/// AST files at this time.
-const unsigned VERSION_MAJOR = 24;
+const unsigned VERSION_MAJOR = 25;
/// AST file minor version number supported by this version of
/// Clang.
@@ -397,6 +397,9 @@ enum UnhashedControlBlockRecordTypes {
/// Record code for the diagnostic options table.
DIAGNOSTIC_OPTIONS,
+ /// Record code for the headers search paths.
+ HEADER_SEARCH_PATHS,
+
/// Record code for \#pragma diagnostic mappings.
DIAG_PRAGMA_MAPPINGS,
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 21cbe4f5b1a5..1b9b12cc5c9c 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -164,6 +164,10 @@ class ASTReaderListener {
/// Receives the header search options.
///
+ /// \param HSOpts The read header search options. The following fields are
+ /// missing and are reported in ReadHeaderSearchPaths():
+ /// UserEntries, SystemHeaderPrefixes, VFSOverlayFiles.
+ ///
/// \returns true to indicate the header search options are invalid, or false
/// otherwise.
virtual bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
@@ -172,6 +176,20 @@ class ASTReaderListener {
return false;
}
+ /// Receives the header search paths.
+ ///
+ /// \param HSOpts The read header search paths. Only the following fields are
+ /// initialized: UserEntries, SystemHeaderPrefixes,
+ /// VFSOverlayFiles. The rest is reported in
+ /// ReadHeaderSearchOptions().
+ ///
+ /// \returns true to indicate the header search paths are invalid, or false
+ /// otherwise.
+ virtual bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
+ bool Complain) {
+ return false;
+ }
+
/// Receives the preprocessor options.
///
/// \param SuggestedPredefines Can be filled in with the set of predefines
@@ -1359,6 +1377,8 @@ class ASTReader
ASTReaderListener &Listener);
static bool ParseHeaderSearchOptions(const RecordData &Record, bool Complain,
ASTReaderListener &Listener);
+ static bool ParseHeaderSearchPaths(const RecordData &Record, bool Complain,
+ ASTReaderListener &Listener);
static bool ParsePreprocessorOptions(const RecordData &Record, bool Complain,
ASTReaderListener &Listener,
std::string &SuggestedPredefines);
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index 382789196add..c5c5bb7d1830 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -87,6 +87,7 @@
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SaveAndRestore.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
@@ -524,6 +525,7 @@ class ASTInfoCollector : public ASTReaderListener {
IntrusiveRefCntPtr<TargetInfo> &Target;
unsigned &Counter;
bool InitializedLanguage = false;
+ bool InitializedHeaderSearchPaths = false;
public:
ASTInfoCollector(Preprocessor &PP, ASTContext *Context,
@@ -550,7 +552,34 @@ class ASTInfoCollector : public ASTReaderListener {
bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
StringRef SpecificModuleCachePath,
bool Complain) override {
+ // Preserve previously set header search paths.
+ llvm::SaveAndRestore X(this->HSOpts.UserEntries);
+ llvm::SaveAndRestore Y(this->HSOpts.SystemHeaderPrefixes);
+ llvm::SaveAndRestore Z(this->HSOpts.VFSOverlayFiles);
+
this->HSOpts = HSOpts;
+
+ return false;
+ }
+
+ bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
+ bool Complain) override {
+ if (InitializedHeaderSearchPaths)
+ return false;
+
+ this->HSOpts.UserEntries = HSOpts.UserEntries;
+ this->HSOpts.SystemHeaderPrefixes = HSOpts.SystemHeaderPrefixes;
+ this->HSOpts.VFSOverlayFiles = HSOpts.VFSOverlayFiles;
+
+ // Initialize the FileManager. We can't do this in update(), since that
+ // performs the initialization too late (once both target and language
+ // options are read).
+ PP.getFileManager().setVirtualFileSystem(createVFSFromOverlayFiles(
+ HSOpts.VFSOverlayFiles, PP.getDiagnostics(),
+ PP.getFileManager().getVirtualFileSystemPtr()));
+
+ InitializedHeaderSearchPaths = true;
+
return false;
}
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 953631955cbb..a637b929782c 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -4737,12 +4737,19 @@ IntrusiveRefCntPtr<llvm::vfs::FileSystem>
clang::createVFSFromCompilerInvocation(
const CompilerInvocation &CI, DiagnosticsEngine &Diags,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) {
- if (CI.getHeaderSearchOpts().VFSOverlayFiles.empty())
+ return createVFSFromOverlayFiles(CI.getHeaderSearchOpts().VFSOverlayFiles,
+ Diags, std::move(BaseFS));
+}
+
+IntrusiveRefCntPtr<llvm::vfs::FileSystem> clang::createVFSFromOverlayFiles(
+ ArrayRef<std::string> VFSOverlayFiles, DiagnosticsEngine &Diags,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) {
+ if (VFSOverlayFiles.empty())
return BaseFS;
IntrusiveRefCntPtr<llvm::vfs::FileSystem> Result = BaseFS;
// earlier vfs files are on the bottom
- for (const auto &File : CI.getHeaderSearchOpts().VFSOverlayFiles) {
+ for (const auto &File : VFSOverlayFiles) {
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
Result->getBufferForFile(File);
if (!Buffer) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 60c09cb1806a..143fd301ffab 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -4805,6 +4805,13 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
Result = OutOfDate; // Don't return early. Read the signature.
break;
}
+ case HEADER_SEARCH_PATHS: {
+ bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
+ if (!AllowCompatibleConfigurationMismatch &&
+ ParseHeaderSearchPaths(Record, Complain, *Listener))
+ Result = ConfigurationMismatch;
+ break;
+ }
case DIAG_PRAGMA_MAPPINGS:
if (!F)
break;
@@ -5907,6 +5914,28 @@ bool ASTReader::ParseHeaderSearchOptions(const RecordData &Record,
unsigned Idx = 0;
HSOpts.Sysroot = ReadString(Record, Idx);
+ HSOpts.ResourceDir = ReadString(Record, Idx);
+ HSOpts.ModuleCachePath = ReadString(Record, Idx);
+ HSOpts.ModuleUserBuildPath = ReadString(Record, Idx);
+ HSOpts.DisableModuleHash = Record[Idx++];
+ HSOpts.ImplicitModuleMaps = Record[Idx++];
+ HSOpts.ModuleMapFileHomeIsCwd = Record[Idx++];
+ HSOpts.EnablePrebuiltImplicitModules = Record[Idx++];
+ HSOpts.UseBuiltinIncludes = Record[Idx++];
+ HSOpts.UseStandardSystemIncludes = Record[Idx++];
+ HSOpts.UseStandardCXXIncludes = Record[Idx++];
+ HSOpts.UseLibcxx = Record[Idx++];
+ std::string SpecificModuleCachePath = ReadString(Record, Idx);
+
+ return Listener.ReadHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
+ Complain);
+}
+
+bool ASTReader::ParseHeaderSearchPaths(const RecordData &Record, bool Complain,
+ ASTReaderListener &Listener) {
+ HeaderSearchOptions HSOpts;
+ unsigned Idx = 0;
+
// Include entries.
for (unsigned N = Record[Idx++]; N; --N) {
std::string Path = ReadString(Record, Idx);
@@ -5925,21 +5954,13 @@ bool ASTReader::ParseHeaderSearchOptions(const RecordData &Record,
HSOpts.SystemHeaderPrefixes.emplace_back(std::move(Prefix), IsSystemHeader);
}
- HSOpts.ResourceDir = ReadString(Record, Idx);
- HSOpts.ModuleCachePath = ReadString(Record, Idx);
- HSOpts.ModuleUserBuildPath = ReadString(Record, Idx);
- HSOpts.DisableModuleHash = Record[Idx++];
- HSOpts.ImplicitModuleMaps = Record[Idx++];
- HSOpts.ModuleMapFileHomeIsCwd = Record[Idx++];
- HSOpts.EnablePrebuiltImplicitModules = Record[Idx++];
- HSOpts.UseBuiltinIncludes = Record[Idx++];
- HSOpts.UseStandardSystemIncludes = Record[Idx++];
- HSOpts.UseStandardCXXIncludes = Record[Idx++];
- HSOpts.UseLibcxx = Record[Idx++];
- std::string SpecificModuleCachePath = ReadString(Record, Idx);
+ // VFS overlay files.
+ for (unsigned N = Record[Idx++]; N; --N) {
+ std::string VFSOverlayFile = ReadString(Record, Idx);
+ HSOpts.VFSOverlayFiles.emplace_back(std::move(VFSOverlayFile));
+ }
- return Listener.ReadHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
- Complain);
+ return Listener.ReadHeaderSearchPaths(HSOpts, Complain);
}
bool ASTReader::ParsePreprocessorOptions(const RecordData &Record,
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 458e88d3688d..2c85d62981ec 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1174,6 +1174,35 @@ ASTFileSignature ASTWriter::writeUnhashedControlBlock(Preprocessor &PP,
Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record);
Record.clear();
+ // Header search paths.
+ Record.clear();
+ const HeaderSearchOptions &HSOpts =
+ PP.getHeaderSearchInfo().getHeaderSearchOpts();
+
+ // Include entries.
+ Record.push_back(HSOpts.UserEntries.size());
+ for (unsigned I = 0, N = HSOpts.UserEntries.size(); I != N; ++I) {
+ const HeaderSearchOptions::Entry &Entry = HSOpts.UserEntries[I];
+ AddString(Entry.Path, Record);
+ Record.push_back(static_cast<unsigned>(Entry.Group));
+ Record.push_back(Entry.IsFramework);
+ Record.push_back(Entry.IgnoreSysRoot);
+ }
+
+ // System header prefixes.
+ Record.push_back(HSOpts.SystemHeaderPrefixes.size());
+ for (unsigned I = 0, N = HSOpts.SystemHeaderPrefixes.size(); I != N; ++I) {
+ AddString(HSOpts.SystemHeaderPrefixes[I].Prefix, Record);
+ Record.push_back(HSOpts.SystemHeaderPrefixes[I].IsSystemHeader);
+ }
+
+ // VFS overlay files.
+ Record.push_back(HSOpts.VFSOverlayFiles.size());
+ for (StringRef VFSOverlayFile : HSOpts.VFSOverlayFiles)
+ AddString(VFSOverlayFile, Record);
+
+ Stream.EmitRecord(HEADER_SEARCH_PATHS, Record);
+
// Write out the diagnostic/pragma mappings.
WritePragmaDiagnosticMappings(Diags, /* isModule = */ WritingModule);
@@ -1399,27 +1428,10 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
// Header search options.
Record.clear();
- const HeaderSearchOptions &HSOpts
- = PP.getHeaderSearchInfo().getHeaderSearchOpts();
- AddString(HSOpts.Sysroot, Record);
-
- // Include entries.
- Record.push_back(HSOpts.UserEntries.size());
- for (unsigned I = 0, N = HSOpts.UserEntries.size(); I != N; ++I) {
- const HeaderSearchOptions::Entry &Entry = HSOpts.UserEntries[I];
- AddString(Entry.Path, Record);
- Record.push_back(static_cast<unsigned>(Entry.Group));
- Record.push_back(Entry.IsFramework);
- Record.push_back(Entry.IgnoreSysRoot);
- }
-
- // System header prefixes.
- Record.push_back(HSOpts.SystemHeaderPrefixes.size());
- for (unsigned I = 0, N = HSOpts.SystemHeaderPrefixes.size(); I != N; ++I) {
- AddString(HSOpts.SystemHeaderPrefixes[I].Prefix, Record);
- Record.push_back(HSOpts.SystemHeaderPrefixes[I].IsSystemHeader);
- }
+ const HeaderSearchOptions &HSOpts =
+ PP.getHeaderSearchInfo().getHeaderSearchOpts();
+ AddString(HSOpts.Sysroot, Record);
AddString(HSOpts.ResourceDir, Record);
AddString(HSOpts.ModuleCachePath, Record);
AddString(HSOpts.ModuleUserBuildPath, Record);
More information about the cfe-commits
mailing list