[clang] [llvm] [clang][DependencyScanner] Remove unused -ivfsoverlay files (PR #73734)
Michael Spencer via cfe-commits
cfe-commits at lists.llvm.org
Thu Jan 25 16:09:24 PST 2024
https://github.com/Bigcheese updated https://github.com/llvm/llvm-project/pull/73734
>From 0559ec44d2d3c39292bae6d6431dde9626ac755e Mon Sep 17 00:00:00 2001
From: Michael Spencer <michael_spencer at apple.com>
Date: Fri, 24 Feb 2023 17:18:51 -0800
Subject: [PATCH] [clang][DepScan] Remove unused -ivfsoverlay files
`-ivfsoverlay` files are unused when building most modules. Enable
removing them by,
* adding a way to visit the filesystem tree with extensible RTTI to
access each `RedirectingFileSystem`.
* Adding tracking to `RedirectingFileSystem` to record when it
actually redirects a file access.
* Storing this information in each PCM.
Usage tracking is only enabled when iterating over the source manager
and affecting modulemaps. Here each path is stated to cause an access.
During scanning these stats all hit the cache.
---
.../Basic/DiagnosticSerializationKinds.td | 3 +
clang/include/clang/Basic/FileManager.h | 4 +
clang/include/clang/Lex/HeaderSearch.h | 6 +
clang/include/clang/Lex/HeaderSearchOptions.h | 6 +-
.../include/clang/Serialization/ASTBitCodes.h | 3 +
clang/include/clang/Serialization/ASTReader.h | 16 +-
.../include/clang/Serialization/ModuleFile.h | 3 +
.../DependencyScanningFilesystem.h | 6 +-
.../DependencyScanningService.h | 11 +-
clang/lib/Basic/FileManager.cpp | 7 +
clang/lib/Frontend/CompilerInvocation.cpp | 1 +
clang/lib/Lex/HeaderSearch.cpp | 16 ++
clang/lib/Serialization/ASTReader.cpp | 41 ++--
clang/lib/Serialization/ASTWriter.cpp | 41 +++-
.../DependencyScanningFilesystem.cpp | 6 +-
.../DependencyScanningWorker.cpp | 86 ++++++--
.../DependencyScanning/ModuleDepCollector.cpp | 74 +++++--
.../ClangScanDeps/optimize-vfs-edgecases.m | 84 ++++++++
clang/test/ClangScanDeps/optimize-vfs-leak.m | 105 ++++++++++
clang/test/ClangScanDeps/optimize-vfs-pch.m | 129 ++++++++++++
clang/test/ClangScanDeps/optimize-vfs.m | 193 ++++++++++++++++++
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 1 +
llvm/include/llvm/Support/VirtualFileSystem.h | 55 ++++-
llvm/lib/Support/VirtualFileSystem.cpp | 26 +++
.../Support/VirtualFileSystemTest.cpp | 86 ++++++++
25 files changed, 932 insertions(+), 77 deletions(-)
create mode 100644 clang/test/ClangScanDeps/optimize-vfs-edgecases.m
create mode 100644 clang/test/ClangScanDeps/optimize-vfs-leak.m
create mode 100644 clang/test/ClangScanDeps/optimize-vfs-pch.m
create mode 100644 clang/test/ClangScanDeps/optimize-vfs.m
diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td
index 11c706ebf84b542..4c4659ed517e0a0 100644
--- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td
@@ -44,6 +44,9 @@ def err_pch_diagopt_mismatch : Error<"%0 is currently enabled, but was not in "
"the PCH file">;
def err_pch_modulecache_mismatch : Error<"PCH was compiled with module cache "
"path '%0', but the path is currently '%1'">;
+def err_pch_vfsoverlay_mismatch : Error<"PCH was compiled with different VFS overlay files than are currently in use">;
+def note_pch_vfsoverlay_files : Note<"%select{PCH|current translation unit}0 has the following VFS overlays:\n%1">;
+def note_pch_vfsoverlay_empty : Note<"%select{PCH|current translation unit}0 has no VFS overlays">;
def err_pch_version_too_old : Error<
"PCH file uses an older PCH format that is no longer supported">;
diff --git a/clang/include/clang/Basic/FileManager.h b/clang/include/clang/Basic/FileManager.h
index 56cb093dd8c376f..997c17a0ffcfcce 100644
--- a/clang/include/clang/Basic/FileManager.h
+++ b/clang/include/clang/Basic/FileManager.h
@@ -248,6 +248,10 @@ class FileManager : public RefCountedBase<FileManager> {
return FS;
}
+ /// Enable or disable tracking of VFS usage. Used to not track full header
+ /// search and implicit modulemap lookup.
+ void trackVFSUsage(bool Active);
+
void setVirtualFileSystem(IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
this->FS = std::move(FS);
}
diff --git a/clang/include/clang/Lex/HeaderSearch.h b/clang/include/clang/Lex/HeaderSearch.h
index a2c33842924b101..49e1a4a124b55df 100644
--- a/clang/include/clang/Lex/HeaderSearch.h
+++ b/clang/include/clang/Lex/HeaderSearch.h
@@ -576,6 +576,12 @@ class HeaderSearch {
/// Note: implicit module maps don't contribute to entry usage.
std::vector<bool> computeUserEntryUsage() const;
+ /// Determine which HeaderSearchOptions::VFSOverlayFiles have been
+ /// successfully used so far and mark their index with 'true' in the resulting
+ /// bit vector.
+ /// Note: implicit module maps don't contribute to entry usage.
+ std::vector<bool> computeVFSUsageAndClear() const;
+
/// This method returns a HeaderMap for the specified
/// FileEntry, uniquing them through the 'HeaderMaps' datastructure.
const HeaderMap *CreateHeaderMap(FileEntryRef FE);
diff --git a/clang/include/clang/Lex/HeaderSearchOptions.h b/clang/include/clang/Lex/HeaderSearchOptions.h
index fa2d0b502d72c19..637dc77e5d957ec 100644
--- a/clang/include/clang/Lex/HeaderSearchOptions.h
+++ b/clang/include/clang/Lex/HeaderSearchOptions.h
@@ -263,6 +263,10 @@ class HeaderSearchOptions {
LLVM_PREFERRED_TYPE(bool)
unsigned ModulesStrictContextHash : 1;
+ /// Whether to include ivfsoverlay usage information in written AST files.
+ LLVM_PREFERRED_TYPE(bool)
+ unsigned ModulesIncludeVFSUsage : 1;
+
HeaderSearchOptions(StringRef _Sysroot = "/")
: Sysroot(_Sysroot), ModuleFormat("raw"), DisableModuleHash(false),
ImplicitModuleMaps(false), ModuleMapFileHomeIsCwd(false),
@@ -277,7 +281,7 @@ class HeaderSearchOptions {
ModulesSkipDiagnosticOptions(false),
ModulesSkipHeaderSearchPaths(false),
ModulesSkipPragmaDiagnosticMappings(false), ModulesHashContent(false),
- ModulesStrictContextHash(false) {}
+ ModulesStrictContextHash(false), ModulesIncludeVFSUsage(false) {}
/// AddPath - Add the \p Path path to the specified \p Group list.
void AddPath(StringRef Path, frontend::IncludeDirGroup Group,
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index fdd64f2abbe9375..f4abfe6f560664f 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -405,6 +405,9 @@ enum UnhashedControlBlockRecordTypes {
/// Record code for the indices of used header search entries.
HEADER_SEARCH_ENTRY_USAGE,
+
+ /// Record code for the indices of used VFSs.
+ VFS_USAGE,
};
/// Record code for extension blocks.
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index dd1451bbf2d2c9f..c09869934b5072e 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -1780,12 +1780,13 @@ class ASTReader
/// Read the control block for the named AST file.
///
/// \returns true if an error occurred, false otherwise.
- static bool readASTFileControlBlock(StringRef Filename, FileManager &FileMgr,
- const InMemoryModuleCache &ModuleCache,
- const PCHContainerReader &PCHContainerRdr,
- bool FindModuleFileExtensions,
- ASTReaderListener &Listener,
- bool ValidateDiagnosticOptions);
+ static bool readASTFileControlBlock(
+ StringRef Filename, FileManager &FileMgr,
+ const InMemoryModuleCache &ModuleCache,
+ const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
+ ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
+ unsigned DefaultClientLoadCapabilities = ARR_ConfigurationMismatch |
+ ARR_OutOfDate);
/// Determine whether the given AST file is acceptable to load into a
/// translation unit with the given language and target options.
@@ -2270,6 +2271,9 @@ class ASTReader
SourceRange ReadSourceRange(ModuleFile &F, const RecordData &Record,
unsigned &Idx, LocSeq *Seq = nullptr);
+ static llvm::BitVector ReadBitVector(const RecordData &Record,
+ const StringRef Blob);
+
// Read a string
static std::string ReadString(const RecordDataImpl &Record, unsigned &Idx);
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 9a14129d72ff33a..bc0aa89966c2b4f 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -189,6 +189,9 @@ class ModuleFile {
/// The bit vector denoting usage of each header search entry (true = used).
llvm::BitVector SearchPathUsage;
+ /// The bit vector denoting usage of each VFS entry (true = used).
+ llvm::BitVector VFSUsage;
+
/// Whether this module has been directly imported by the
/// user.
bool DirectlyImported = false;
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 9a2aea5d6efa170..846fdc7253977f9 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -280,8 +280,12 @@ class EntryRef {
/// This is not a thread safe VFS. A single instance is meant to be used only in
/// one thread. Multiple instances are allowed to service multiple threads
/// running in parallel.
-class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
+class DependencyScanningWorkerFilesystem
+ : public llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
+ llvm::vfs::ProxyFileSystem> {
public:
+ static const char ID;
+
DependencyScanningWorkerFilesystem(
DependencyScanningFilesystemSharedCache &SharedCache,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
index dcdf1c171f6d731..4f9867262a275c1 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -45,6 +45,9 @@ enum class ScanningOutputFormat {
P1689,
};
+#define DSS_LAST_BITMASK_ENUM(Id) \
+ LLVM_MARK_AS_BITMASK_ENUM(Id), All = llvm::NextPowerOf2(Id) - 1
+
enum class ScanningOptimizations {
None = 0,
@@ -54,11 +57,15 @@ enum class ScanningOptimizations {
/// Remove warnings from system modules.
SystemWarnings = 2,
- LLVM_MARK_AS_BITMASK_ENUM(SystemWarnings),
- All = HeaderSearch | SystemWarnings,
+ /// Remove unused -ivfsoverlay arguments.
+ VFS = 4,
+
+ DSS_LAST_BITMASK_ENUM(VFS),
Default = All
};
+#undef DSS_LAST_BITMASK_ENUM
+
/// The dependency scanning service contains shared configuration and state that
/// is used by the individual dependency scanning workers.
class DependencyScanningService {
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index 4e77b178a86b9ee..6097a27e429d661 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -363,6 +363,13 @@ llvm::Expected<FileEntryRef> FileManager::getSTDIN() {
return *STDIN;
}
+void FileManager::trackVFSUsage(bool Active) {
+ FS->visit([Active](llvm::vfs::FileSystem &FileSys) {
+ if (auto *RFS = dyn_cast<llvm::vfs::RedirectingFileSystem>(&FileSys))
+ RFS->setUsageTrackingActive(Active);
+ });
+}
+
const FileEntry *FileManager::getVirtualFile(StringRef Filename, off_t Size,
time_t ModificationTime) {
return &getVirtualFileRef(Filename, Size, ModificationTime).getFileEntry();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index feb4de2084b830b..8d7b75b56d61290 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -4763,6 +4763,7 @@ std::string CompilerInvocation::getModuleHash() const {
if (hsOpts.ModulesStrictContextHash) {
HBuilder.addRange(hsOpts.SystemHeaderPrefixes);
HBuilder.addRange(hsOpts.UserEntries);
+ HBuilder.addRange(hsOpts.VFSOverlayFiles);
const DiagnosticOptions &diagOpts = getDiagnosticOpts();
#define DIAGOPT(Name, Bits, Default) HBuilder.add(diagOpts.Name);
diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
index 0f1090187734ff2..e372671a19d44d4 100644
--- a/clang/lib/Lex/HeaderSearch.cpp
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -141,6 +141,22 @@ std::vector<bool> HeaderSearch::computeUserEntryUsage() const {
return UserEntryUsage;
}
+std::vector<bool> HeaderSearch::computeVFSUsageAndClear() const {
+ std::vector<bool> VFSUsage;
+ llvm::vfs::FileSystem &RootFS = FileMgr.getVirtualFileSystem();
+ // TODO: This only works if the `RedirectingFileSystem`s were all created by
+ // `createVFSFromOverlayFiles`.
+ RootFS.visit([&](llvm::vfs::FileSystem &FS) {
+ if (auto *RFS = dyn_cast<llvm::vfs::RedirectingFileSystem>(&FS)) {
+ VFSUsage.push_back(RFS->hasBeenUsed());
+ RFS->clearHasBeenUsed();
+ }
+ });
+ // VFS visit order is the opposite of VFSOverlayFiles order.
+ std::reverse(VFSUsage.begin(), VFSUsage.end());
+ return VFSUsage;
+}
+
/// CreateHeaderMap - This method returns a HeaderMap for the specified
/// FileEntry, uniquing them through the 'HeaderMaps' datastructure.
const HeaderMap *HeaderSearch::CreateHeaderMap(FileEntryRef FE) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index fecd94e875f671a..297ac35ef7689a0 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -4977,7 +4977,7 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
}
case HEADER_SEARCH_PATHS: {
bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
- if (!AllowCompatibleConfigurationMismatch &&
+ if (Listener && !AllowCompatibleConfigurationMismatch &&
ParseHeaderSearchPaths(Record, Complain, *Listener))
Result = ConfigurationMismatch;
break;
@@ -4991,18 +4991,17 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
F->PragmaDiagMappings.insert(F->PragmaDiagMappings.end(),
Record.begin(), Record.end());
break;
- case HEADER_SEARCH_ENTRY_USAGE:
- if (!F)
- break;
- unsigned Count = Record[0];
- const char *Byte = Blob.data();
- F->SearchPathUsage = llvm::BitVector(Count, false);
- for (unsigned I = 0; I < Count; ++Byte)
- for (unsigned Bit = 0; Bit < 8 && I < Count; ++Bit, ++I)
- if (*Byte & (1 << Bit))
- F->SearchPathUsage[I] = true;
+ case HEADER_SEARCH_ENTRY_USAGE: {
+ if (F)
+ F->SearchPathUsage = ReadBitVector(Record, Blob);
+ break;
+ }
+ case VFS_USAGE: {
+ if (F)
+ F->VFSUsage = ReadBitVector(Record, Blob);
break;
}
+ }
}
}
@@ -5398,7 +5397,8 @@ bool ASTReader::readASTFileControlBlock(
StringRef Filename, FileManager &FileMgr,
const InMemoryModuleCache &ModuleCache,
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
- ASTReaderListener &Listener, bool ValidateDiagnosticOptions) {
+ ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
+ unsigned DefaultClientLoadCapabilities) {
// Open the AST file.
std::unique_ptr<llvm::MemoryBuffer> OwnedBuffer;
llvm::MemoryBuffer *Buffer = ModuleCache.lookupPCM(Filename);
@@ -5453,7 +5453,7 @@ bool ASTReader::readASTFileControlBlock(
switch (Entry.ID) {
case OPTIONS_BLOCK_ID: {
std::string IgnoredSuggestedPredefines;
- if (ReadOptionsBlock(Stream, ARR_ConfigurationMismatch | ARR_OutOfDate,
+ if (ReadOptionsBlock(Stream, DefaultClientLoadCapabilities,
/*AllowCompatibleConfigurationMismatch*/ false,
Listener, IgnoredSuggestedPredefines) != Success)
return true;
@@ -5679,7 +5679,7 @@ bool ASTReader::readASTFileControlBlock(
// Scan for the UNHASHED_CONTROL_BLOCK_ID block.
if (readUnhashedControlBlockImpl(
- nullptr, Bytes, ARR_ConfigurationMismatch | ARR_OutOfDate,
+ nullptr, Bytes, DefaultClientLoadCapabilities,
/*AllowCompatibleConfigurationMismatch*/ false, &Listener,
ValidateDiagnosticOptions) != Success)
return true;
@@ -9312,6 +9312,19 @@ SourceRange ASTReader::ReadSourceRange(ModuleFile &F, const RecordData &Record,
return SourceRange(beg, end);
}
+llvm::BitVector ASTReader::ReadBitVector(const RecordData &Record,
+ const StringRef Blob) {
+ llvm::BitVector Ret;
+ unsigned Count = Record[0];
+ const char *Byte = Blob.data();
+ Ret = llvm::BitVector(Count, false);
+ for (unsigned I = 0; I < Count; ++Byte)
+ for (unsigned Bit = 0; Bit < 8 && I < Count; ++Bit, ++I)
+ if (*Byte & (1 << Bit))
+ Ret[I] = true;
+ return Ret;
+}
+
/// Read a floating-point value
llvm::APFloat ASTRecordReader::readAPFloat(const llvm::fltSemantics &Sem) {
return llvm::APFloat(Sem, readAPInt());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 03bddfe0f5047db..26f2954a45019b8 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1265,18 +1265,30 @@ void ASTWriter::writeUnhashedControlBlock(Preprocessor &PP,
WritePragmaDiagnosticMappings(Diags, /* isModule = */ WritingModule);
// Header search entry usage.
- auto HSEntryUsage = PP.getHeaderSearchInfo().computeUserEntryUsage();
- auto Abbrev = std::make_shared<BitCodeAbbrev>();
- Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_ENTRY_USAGE));
- Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
- Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
- unsigned HSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
{
+ auto HSEntryUsage = PP.getHeaderSearchInfo().computeUserEntryUsage();
+ auto Abbrev = std::make_shared<BitCodeAbbrev>();
+ Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_ENTRY_USAGE));
+ Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
+ Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
+ unsigned HSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
RecordData::value_type Record[] = {HEADER_SEARCH_ENTRY_USAGE,
HSEntryUsage.size()};
Stream.EmitRecordWithBlob(HSUsageAbbrevCode, Record, bytes(HSEntryUsage));
}
+ // VFS usage.
+ {
+ auto VFSUsage = PP.getHeaderSearchInfo().computeVFSUsageAndClear();
+ auto Abbrev = std::make_shared<BitCodeAbbrev>();
+ Abbrev->Add(BitCodeAbbrevOp(VFS_USAGE));
+ Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
+ Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
+ unsigned VFSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
+ RecordData::value_type Record[] = {VFS_USAGE, VFSUsage.size()};
+ Stream.EmitRecordWithBlob(VFSUsageAbbrevCode, Record, bytes(VFSUsage));
+ }
+
// Leave the options block.
Stream.ExitBlock();
UnhashedControlBlockRange.second = Stream.GetCurrentBitNo() >> 3;
@@ -4687,6 +4699,15 @@ void ASTWriter::collectNonAffectingInputFiles() {
NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
+ const bool IncludeVFSUsage =
+ PP->getHeaderSearchInfo().getHeaderSearchOpts().ModulesIncludeVFSUsage;
+ FileManager &FileMgr = PP->getFileManager();
+ FileMgr.trackVFSUsage(true);
+ if (IncludeVFSUsage)
+ for (StringRef Path :
+ PP->getHeaderSearchInfo().getHeaderSearchOpts().VFSOverlayFiles)
+ FileMgr.getVirtualFileSystem().exists(Path);
+
for (unsigned I = 1; I != N; ++I) {
const SrcMgr::SLocEntry *SLoc = &SrcMgr.getLocalSLocEntry(I);
FileID FID = FileID::get(I);
@@ -4701,8 +4722,13 @@ void ASTWriter::collectNonAffectingInputFiles() {
if (!isModuleMap(File.getFileCharacteristic()) ||
AffectingModuleMaps.empty() ||
- llvm::is_contained(AffectingModuleMaps, *Cache->OrigEntry))
+ llvm::is_contained(AffectingModuleMaps, *Cache->OrigEntry)) {
+ // Lookup the path in the VFS to trigger `-ivfsoverlay` usage tracking.
+ if (IncludeVFSUsage)
+ FileMgr.getVirtualFileSystem().exists(
+ File.getContentCache().OrigEntry->getNameAsRequested());
continue;
+ }
IsSLocAffecting[I] = false;
@@ -4727,6 +4753,7 @@ void ASTWriter::collectNonAffectingInputFiles() {
NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
}
+ FileMgr.trackVFSUsage(false);
}
ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 6f71650a3982c08..1b750cec41e1cc4 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -194,7 +194,9 @@ static bool shouldCacheStatFailures(StringRef Filename) {
DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
DependencyScanningFilesystemSharedCache &SharedCache,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
- : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
+ : llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
+ llvm::vfs::ProxyFileSystem>(std::move(FS)),
+ SharedCache(SharedCache),
WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
updateWorkingDirForCacheLookup();
}
@@ -379,3 +381,5 @@ void DependencyScanningWorkerFilesystem::updateWorkingDirForCacheLookup() {
assert(!WorkingDirForCacheLookup ||
llvm::sys::path::is_absolute_gnu(*WorkingDirForCacheLookup));
}
+
+const char DependencyScanningWorkerFilesystem::ID = 0;
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 7ab4a699af6dfca..390cbe5aa65e1b8 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -9,6 +9,7 @@
#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Basic/DiagnosticDriver.h"
#include "clang/Basic/DiagnosticFrontend.h"
+#include "clang/Basic/DiagnosticSerialization.h"
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
@@ -59,6 +60,31 @@ class DependencyConsumerForwarder : public DependencyFileGenerator {
DependencyConsumer &C;
};
+static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
+ const HeaderSearchOptions &ExistingHSOpts,
+ DiagnosticsEngine *Diags,
+ const LangOptions &LangOpts) {
+ if (LangOpts.Modules) {
+ if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
+ if (Diags) {
+ Diags->Report(diag::err_pch_vfsoverlay_mismatch);
+ auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
+ if (VFSOverlays.empty()) {
+ Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;
+ } else {
+ std::string Files = llvm::join(VFSOverlays, "\n");
+ Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;
+ }
+ };
+ VFSNote(0, HSOpts.VFSOverlayFiles);
+ VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
+ }
+ return true;
+ }
+ }
+ return false;
+}
+
using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
/// A listener that collects the imported modules and optionally the input
@@ -66,9 +92,12 @@ using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
class PrebuiltModuleListener : public ASTReaderListener {
public:
PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
- llvm::SmallVector<std::string> &NewModuleFiles)
+ llvm::SmallVector<std::string> &NewModuleFiles,
+ const HeaderSearchOptions &HSOpts,
+ const LangOptions &LangOpts, DiagnosticsEngine &Diags)
: PrebuiltModuleFiles(PrebuiltModuleFiles),
- NewModuleFiles(NewModuleFiles) {}
+ NewModuleFiles(NewModuleFiles), ExistingHSOpts(HSOpts),
+ ExistingLangOpts(LangOpts), Diags(Diags) {}
bool needsImportVisitation() const override { return true; }
@@ -77,26 +106,47 @@ class PrebuiltModuleListener : public ASTReaderListener {
NewModuleFiles.push_back(Filename.str());
}
+ bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
+ bool Complain) override {
+ return checkHeaderSearchPaths(
+ HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
+ }
+
private:
PrebuiltModuleFilesT &PrebuiltModuleFiles;
llvm::SmallVector<std::string> &NewModuleFiles;
+ const HeaderSearchOptions &ExistingHSOpts;
+ const LangOptions &ExistingLangOpts;
+ DiagnosticsEngine &Diags;
};
/// Visit the given prebuilt module and collect all of the modules it
/// transitively imports and contributing input files.
-static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
+static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
CompilerInstance &CI,
- PrebuiltModuleFilesT &ModuleFiles) {
+ PrebuiltModuleFilesT &ModuleFiles,
+ DiagnosticsEngine &Diags) {
// List of module files to be processed.
- llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
- PrebuiltModuleListener Listener(ModuleFiles, Worklist);
-
- while (!Worklist.empty())
- ASTReader::readASTFileControlBlock(
- Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
- CI.getPCHContainerReader(),
- /*FindModuleFileExtensions=*/false, Listener,
- /*ValidateDiagnosticOptions=*/false);
+ llvm::SmallVector<std::string> Worklist;
+ PrebuiltModuleListener Listener(
+ ModuleFiles, Worklist, CI.getHeaderSearchOpts(), CI.getLangOpts(), Diags);
+
+ if (ASTReader::readASTFileControlBlock(
+ PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),
+ CI.getPCHContainerReader(),
+ /*FindModuleFileExtensions=*/false, Listener,
+ /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))
+ return true;
+
+ while (!Worklist.empty()) {
+ if (ASTReader::readASTFileControlBlock(
+ Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
+ CI.getPCHContainerReader(),
+ /*FindModuleFileExtensions=*/false, Listener,
+ /*ValidateDiagnosticOptions=*/false))
+ return true;
+ }
+ return false;
}
/// Transform arbitrary file name into an object-like file name.
@@ -183,6 +233,7 @@ class DependencyScanningAction : public tooling::ToolAction {
ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
ScanInstance.getFrontendOpts().ModulesShareFileManager = false;
ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";
+ ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage = true;
ScanInstance.setFileManager(FileMgr);
// Support for virtual file system overlays.
@@ -196,9 +247,12 @@ class DependencyScanningAction : public tooling::ToolAction {
// will prevent the implicit build to create duplicate modules and will
// force reuse of the existing prebuilt module files instead.
if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
- visitPrebuiltModule(
- ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
- ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles);
+ if (visitPrebuiltModule(
+ ScanInstance.getPreprocessorOpts().ImplicitPCHInclude,
+ ScanInstance,
+ ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
+ ScanInstance.getDiagnostics()))
+ return false;
// Use the dependency scanning optimized file system if requested to do so.
if (DepFS) {
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index bfaa897851041db..b807dc843218523 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -31,25 +31,55 @@ const std::vector<std::string> &ModuleDeps::getBuildArguments() {
static void optimizeHeaderSearchOpts(HeaderSearchOptions &Opts,
ASTReader &Reader,
- const serialization::ModuleFile &MF) {
- // Only preserve search paths that were used during the dependency scan.
- std::vector<HeaderSearchOptions::Entry> Entries = Opts.UserEntries;
- Opts.UserEntries.clear();
-
- llvm::BitVector SearchPathUsage(Entries.size());
- llvm::DenseSet<const serialization::ModuleFile *> Visited;
- std::function<void(const serialization::ModuleFile *)> VisitMF =
- [&](const serialization::ModuleFile *MF) {
- SearchPathUsage |= MF->SearchPathUsage;
- Visited.insert(MF);
- for (const serialization::ModuleFile *Import : MF->Imports)
- if (!Visited.contains(Import))
- VisitMF(Import);
- };
- VisitMF(&MF);
-
- for (auto Idx : SearchPathUsage.set_bits())
- Opts.UserEntries.push_back(Entries[Idx]);
+ const serialization::ModuleFile &MF,
+ ScanningOptimizations OptimizeArgs) {
+ if (any(OptimizeArgs & ScanningOptimizations::HeaderSearch)) {
+ // Only preserve search paths that were used during the dependency scan.
+ std::vector<HeaderSearchOptions::Entry> Entries;
+ std::swap(Opts.UserEntries, Entries);
+
+ llvm::BitVector SearchPathUsage(Entries.size());
+ llvm::DenseSet<const serialization::ModuleFile *> Visited;
+ std::function<void(const serialization::ModuleFile *)> VisitMF =
+ [&](const serialization::ModuleFile *MF) {
+ SearchPathUsage |= MF->SearchPathUsage;
+ Visited.insert(MF);
+ for (const serialization::ModuleFile *Import : MF->Imports)
+ if (!Visited.contains(Import))
+ VisitMF(Import);
+ };
+ VisitMF(&MF);
+
+ if (SearchPathUsage.size() != Entries.size())
+ llvm::report_fatal_error(
+ "Inconsistent search path options between modules detected");
+
+ for (auto Idx : SearchPathUsage.set_bits())
+ Opts.UserEntries.push_back(std::move(Entries[Idx]));
+ }
+ if (any(OptimizeArgs & ScanningOptimizations::VFS)) {
+ std::vector<std::string> VFSOverlayFiles;
+ std::swap(Opts.VFSOverlayFiles, VFSOverlayFiles);
+
+ llvm::BitVector VFSUsage(VFSOverlayFiles.size());
+ llvm::DenseSet<const serialization::ModuleFile *> Visited;
+ std::function<void(const serialization::ModuleFile *)> VisitMF =
+ [&](const serialization::ModuleFile *MF) {
+ VFSUsage |= MF->VFSUsage;
+ Visited.insert(MF);
+ for (const serialization::ModuleFile *Import : MF->Imports)
+ if (!Visited.contains(Import))
+ VisitMF(Import);
+ };
+ VisitMF(&MF);
+
+ if (VFSUsage.size() != VFSOverlayFiles.size())
+ llvm::report_fatal_error(
+ "Inconsistent -ivfsoverlay options between modules detected");
+
+ for (auto Idx : VFSUsage.set_bits())
+ Opts.VFSOverlayFiles.push_back(std::move(VFSOverlayFiles[Idx]));
+ }
}
static void optimizeDiagnosticOpts(DiagnosticOptions &Opts,
@@ -558,9 +588,11 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
CowCompilerInvocation CI =
MDC.getInvocationAdjustedForModuleBuildWithoutOutputs(
MD, [&](CowCompilerInvocation &BuildInvocation) {
- if (any(MDC.OptimizeArgs & ScanningOptimizations::HeaderSearch))
+ if (any(MDC.OptimizeArgs & (ScanningOptimizations::HeaderSearch |
+ ScanningOptimizations::VFS)))
optimizeHeaderSearchOpts(BuildInvocation.getMutHeaderSearchOpts(),
- *MDC.ScanInstance.getASTReader(), *MF);
+ *MDC.ScanInstance.getASTReader(), *MF,
+ MDC.OptimizeArgs);
if (any(MDC.OptimizeArgs & ScanningOptimizations::SystemWarnings))
optimizeDiagnosticOpts(
BuildInvocation.getMutDiagnosticOpts(),
diff --git a/clang/test/ClangScanDeps/optimize-vfs-edgecases.m b/clang/test/ClangScanDeps/optimize-vfs-edgecases.m
new file mode 100644
index 000000000000000..0f18571dd05b230
--- /dev/null
+++ b/clang/test/ClangScanDeps/optimize-vfs-edgecases.m
@@ -0,0 +1,84 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands.json.in > %t/build/compile-commands.json
+// RUN: sed -e "s|DIR|%/t|g" %t/build/vfsoverlay.yaml.in > %t/build/vfsoverlay.yaml
+// RUN: sed -e "s|DIR|%/t|g" %t/build/vfs.notyaml.in > %t/build/vfs.notyaml
+// RUN: clang-scan-deps -compilation-database %t/build/compile-commands.json \
+// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search > %t/deps.db
+
+// RUN: %deps-to-rsp %t/deps.db --module-name=A > %t/A.rsp
+// RUN: cd %t && %clang @%t/A.rsp
+
+// Check that the following edge cases are handled by ivfsoverlay tracking
+// * `-ivfsoverlay` args that depend on earlier `-ivfsoverlay` args.
+
+//--- build/compile-commands.json.in
+
+[
+{
+ "directory": "DIR",
+ "command": "clang -c DIR/0.m -Imodules/A -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/vfsoverlay.yaml -ivfsoverlay build/vfs.yaml",
+ "file": "DIR/0.m"
+}
+]
+
+//--- build/vfsoverlay.yaml.in
+
+{
+ "version":0,
+ "case-sensitive":"false",
+ "roots":[
+ {
+ "contents":[
+ {
+ "external-contents":"DIR/build/vfs.notyaml",
+ "name":"vfs.yaml",
+ "type":"file"
+ }
+ ],
+ "name":"DIR/build",
+ "type":"directory"
+ }
+ ]
+}
+
+//--- build/vfs.notyaml.in
+
+{
+ "version":0,
+ "case-sensitive":"false",
+ "roots":[
+ {
+ "contents":[
+ {
+ "external-contents":"DIR/build/module.modulemap",
+ "name":"module.modulemap",
+ "type":"file"
+ },
+ {
+ "external-contents":"DIR/build/A.h",
+ "name":"A.h",
+ "type":"file"
+ }
+ ],
+ "name":"DIR/modules/A",
+ "type":"directory"
+ }
+ ]
+}
+
+//--- build/module.modulemap
+
+module A {
+ umbrella header "A.h"
+}
+
+//--- build/A.h
+
+typedef int A_t;
+
+//--- 0.m
+
+#include <A.h>
+
+A_t a = 0;
diff --git a/clang/test/ClangScanDeps/optimize-vfs-leak.m b/clang/test/ClangScanDeps/optimize-vfs-leak.m
new file mode 100644
index 000000000000000..beda44cdfecb7c9
--- /dev/null
+++ b/clang/test/ClangScanDeps/optimize-vfs-leak.m
@@ -0,0 +1,105 @@
+// This test checks that VFS usage doesn't leak between modules.
+
+// RUN: rm -rf %t && split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/build/cdb.json.in > %t/build/cdb.json
+// RUN: sed -e "s|DIR|%/t|g" %t/build/vfs.yaml.in > %t/build/vfs.yaml
+// RUN: clang-scan-deps -compilation-database %t/build/cdb.json \
+// RUN: -format experimental-full --optimize-args=vfs > %t/deps.json
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+
+// CHECK: {
+// CHECK-NEXT: "modules": [
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang-module-deps": [
+// CHECK-NEXT: {
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "module-name": "B"
+// CHECK-NEXT: },
+// CHECK-NEXT: {
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "module-name": "C"
+// CHECK-NEXT: }
+// CHECK-NEXT: ],
+// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/moduleA/module.modulemap",
+// CHECK-NEXT: "command-line": [
+// Module A needs the VFS overlay because its dependency, module B, needs it.
+// CHECK: "-ivfsoverlay"
+// CHECK-NEXT: "[[PREFIX]]/build/vfs.yaml"
+// CHECK: ],
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "file-deps": [
+// CHECK: ],
+// CHECK-NEXT: "name": "A"
+// CHECK-NEXT: },
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang-module-deps": [],
+// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/moduleB/module.modulemap",
+// CHECK-NEXT: "command-line": [
+// Module B needs the VFS overlay because it provides the header referred to by the module map.
+// CHECK: "-ivfsoverlay"
+// CHECK-NEXT: "[[PREFIX]]/build/vfs.yaml"
+// CHECK: ],
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "file-deps": [
+// CHECK: ],
+// CHECK-NEXT: "name": "B"
+// CHECK-NEXT: },
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang-module-deps": [],
+// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/moduleC/module.modulemap",
+// CHECK-NEXT: "command-line": [
+// Module C doesn't need the VFS overlay.
+// CHECK-NOT: "-ivfsoverlay"
+// CHECK: ],
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "file-deps": [
+// CHECK: ],
+// CHECK-NEXT: "name": "C"
+// CHECK-NEXT: }
+// CHECK-NEXT: ],
+// CHECK-NEXT: "translation-units": [
+// CHECK: ]
+// CHECK: }
+
+//--- build/cdb.json.in
+[{
+ "directory": "DIR",
+ "command": "clang -c DIR/tu.m -I DIR/moduleA -I DIR/moduleB -I DIR/moduleC -fmodules -fmodules-cache-path=DIR/cache -fimplicit-module-maps -ivfsoverlay DIR/build/vfs.yaml",
+ "file": "DIR/tu.m"
+}]
+
+//--- build/vfs.yaml.in
+{
+ "version": 0,
+ "case-sensitive": "false",
+ "roots": [
+ {
+ "contents": [
+ {
+ "external-contents": "DIR/build/B.h",
+ "name": "B.h",
+ "type": "file"
+ }
+ ],
+ "name": "DIR/moduleB",
+ "type": "directory"
+ }
+ ]
+}
+
+//--- tu.m
+ at import A;
+
+//--- moduleA/module.modulemap
+module A { header "A.h" }
+//--- moduleA/A.h
+ at import B;
+ at import C;
+
+//--- moduleB/module.modulemap
+module B { header "B.h" }
+//--- build/B.h
+
+//--- moduleC/module.modulemap
+module C { header "C.h" }
+//--- moduleC/C.h
\ No newline at end of file
diff --git a/clang/test/ClangScanDeps/optimize-vfs-pch.m b/clang/test/ClangScanDeps/optimize-vfs-pch.m
new file mode 100644
index 000000000000000..e6acb73e1dd3439
--- /dev/null
+++ b/clang/test/ClangScanDeps/optimize-vfs-pch.m
@@ -0,0 +1,129 @@
+// Check that tracking of VFSs works with PCH.
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands-pch.json.in > %t/build/compile-commands-pch.json
+// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands-tu.json.in > %t/build/compile-commands-tu.json
+// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands-tu-no-vfs.json.in > %t/build/compile-commands-tu-no-vfs.json
+// RUN: sed -e "s|DIR|%/t|g" %t/build/pch-overlay.yaml.in > %t/build/pch-overlay.yaml
+
+// RUN: clang-scan-deps -compilation-database %t/build/compile-commands-pch.json \
+// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search > %t/pch-deps.db
+// RUN: %deps-to-rsp %t/pch-deps.db --module-name=A > %t/A.rsp
+// RUN: %deps-to-rsp %t/pch-deps.db --module-name=B > %t/B.rsp
+// RUN: %deps-to-rsp %t/pch-deps.db --tu-index=0 > %t/pch.rsp
+// RUN: %clang @%t/A.rsp
+// RUN: %clang @%t/B.rsp
+// RUN: %clang @%t/pch.rsp
+
+// RUN: clang-scan-deps -compilation-database %t/build/compile-commands-tu.json \
+// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search > %t/tu-deps.db
+// RUN: %deps-to-rsp %t/tu-deps.db --module-name=C > %t/C.rsp
+// RUN: %deps-to-rsp %t/tu-deps.db --tu-index=0 > %t/tu.rsp
+// RUN: %clang @%t/C.rsp
+// RUN: %clang @%t/tu.rsp
+
+// RUN: not clang-scan-deps -compilation-database %t/build/compile-commands-tu-no-vfs.json \
+// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search 2>&1 | FileCheck %s
+
+// CHECK: error: PCH was compiled with different VFS overlay files than are currently in use
+// CHECK: note: current translation unit has no VFS overlays
+
+//--- build/compile-commands-pch.json.in
+
+[
+{
+ "directory": "DIR",
+ "command": "clang -x objective-c-header DIR/pch.h -I DIR/modules/A -I DIR/modules/B -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -o DIR/pch.h.pch -ivfsoverlay DIR/build/pch-overlay.yaml",
+ "file": "DIR/pch.h"
+}
+]
+
+//--- build/compile-commands-tu.json.in
+
+[
+{
+ "directory": "DIR",
+ "command": "clang -fsyntax-only DIR/tu.m -I DIR/modules/A -I DIR/modules/B -I DIR/modules/C -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -include DIR/pch.h -o DIR/tu.o -ivfsoverlay DIR/build/pch-overlay.yaml",
+ "file": "DIR/tu.m"
+}
+]
+
+//--- build/compile-commands-tu-no-vfs.json.in
+
+[
+{
+ "directory": "DIR",
+ "command": "clang -fsyntax-only DIR/tu.m -I DIR/modules/A -I DIR/modules/B -I DIR/modules/C -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -include DIR/pch.h -o DIR/tu.o",
+ "file": "DIR/tu.m"
+}
+]
+
+//--- build/pch-overlay.yaml.in
+
+{
+ "version":0,
+ "case-sensitive":"false",
+ "roots":[
+ {
+ "contents":[
+ {
+ "external-contents":"DIR/build/module.modulemap",
+ "name":"module.modulemap",
+ "type":"file"
+ },
+ {
+ "external-contents":"DIR/build/A.h",
+ "name":"A.h",
+ "type":"file"
+ }
+ ],
+ "name":"DIR/modules/A",
+ "type":"directory"
+ }
+ ]
+}
+
+//--- pch.h
+#include <B.h>
+
+//--- build/module.modulemap
+
+module A {
+ umbrella header "A.h"
+}
+
+//--- build/A.h
+
+typedef int A_t;
+
+//--- modules/B/module.modulemap
+
+module B {
+ umbrella header "B.h"
+ export *
+}
+
+//--- modules/B/B.h
+#include <A.h>
+
+typedef int B_t;
+
+//--- modules/C/module.modulemap
+
+module C {
+ umbrella header "C.h"
+}
+
+//--- modules/C/C.h
+#include <B.h>
+
+typedef int C_t;
+
+//--- tu.m
+
+#include <C.h>
+
+A_t a = 0;
+B_t b = 0;
+C_t c = 0;
diff --git a/clang/test/ClangScanDeps/optimize-vfs.m b/clang/test/ClangScanDeps/optimize-vfs.m
new file mode 100644
index 000000000000000..20c97956087d2d8
--- /dev/null
+++ b/clang/test/ClangScanDeps/optimize-vfs.m
@@ -0,0 +1,193 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands.json.in > %t/build/compile-commands.json
+// RUN: sed -e "s|DIR|%/t|g" %t/build/vfs.yaml.in > %t/build/vfs.yaml
+// RUN: sed -e "s|DIR|%/t|g" %t/build/unused-vfs.yaml.in > %t/build/unused-vfs.yaml
+// RUN: sed -e "s|DIR|%/t|g" %t/build/unused-vfs.yaml.in > %t/build/unused2-vfs.yaml
+// RUN: clang-scan-deps -compilation-database %t/build/compile-commands.json \
+// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search > %t/deps.db
+// RUN: cat %t/deps.db | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+
+// Check that unused -ivfsoverlay arguments are removed, and that used ones are
+// not.
+
+// CHECK: {
+// CHECK-NEXT: "modules": [
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang-module-deps": [],
+// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/modules/A/module.modulemap",
+// CHECK-NEXT: "command-line": [
+// CHECK-NOT: "build/unused-vfs.yaml"
+// CHECK: "-ivfsoverlay"
+// CHECK-NEXT: "build/vfs.yaml"
+// CHECK-NOT: "build/unused-vfs.yaml"
+// CHECK: ],
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "file-deps": [
+// CHECK-NEXT: "[[PREFIX]]/build/A.h",
+// CHECK-NEXT: "[[PREFIX]]/build/module.modulemap"
+// CHECK-NEXT: ],
+// CHECK-NEXT: "name": "A"
+// CHECK-NEXT: },
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang-module-deps": [],
+// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/modules/B/module.modulemap",
+// CHECK-NEXT: "command-line": [
+// CHECK-NOT: "-ivfsoverlay"
+// CHECK: ],
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "file-deps": [
+// CHECK-NEXT: "[[PREFIX]]/modules/B/B.h",
+// CHECK-NEXT: "[[PREFIX]]/modules/B/module.modulemap"
+// CHECK-NEXT: ],
+// CHECK-NEXT: "name": "B"
+// CHECK-NEXT: },
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang-module-deps": [
+// CHECK-NEXT: {
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "module-name": "B"
+// CHECK-NEXT: }
+// CHECK-NEXT: ],
+// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/modules/C/module.modulemap",
+// CHECK-NEXT: "command-line": [
+// CHECK-NOT: "-ivfsoverlay"
+// CHECK: ],
+// CHECK-NEXT: "context-hash": "{{.*}}",
+// CHECK-NEXT: "file-deps": [
+// CHECK-NEXT: "[[PREFIX]]/modules/B/module.modulemap",
+// CHECK-NEXT: "[[PREFIX]]/modules/C/C.h",
+// CHECK-NEXT: "[[PREFIX]]/modules/C/module.modulemap"
+// CHECK-NEXT: ],
+// CHECK-NEXT: "name": "C"
+// CHECK-NEXT: }
+// CHECK-NEXT: ],
+// CHECK-NEXT: "translation-units": [
+// CHECK: ]
+// CHECK: }
+
+//--- build/compile-commands.json.in
+
+[
+{
+ "directory": "DIR",
+ "command": "clang -c DIR/0.m -Imodules/A -Imodules/B -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/unused-vfs.yaml -ivfsoverlay build/unused2-vfs.yaml -ivfsoverlay build/vfs.yaml",
+ "file": "DIR/0.m"
+},
+{
+ "directory": "DIR",
+ "command": "clang -c DIR/A.m -Imodules/A -Imodules/B -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/vfs.yaml -ivfsoverlay build/unused-vfs.yaml",
+ "file": "DIR/A.m"
+},
+{
+ "directory": "DIR",
+ "command": "clang -c DIR/B.m -Imodules/B -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/unused-vfs.yaml -ivfsoverlay build/vfs.yaml",
+ "file": "DIR/B.m"
+},
+{
+ "directory": "DIR",
+ "command": "clang -c DIR/C.m -Imodules/A -Imodules/B -Imodules/C -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/vfs.yaml -ivfsoverlay build/unused-vfs.yaml",
+ "file": "DIR/C.m"
+}
+]
+
+//--- build/vfs.yaml.in
+
+{
+ "version":0,
+ "case-sensitive":"false",
+ "roots":[
+ {
+ "contents":[
+ {
+ "external-contents":"DIR/build/module.modulemap",
+ "name":"module.modulemap",
+ "type":"file"
+ },
+ {
+ "external-contents":"DIR/build/A.h",
+ "name":"A.h",
+ "type":"file"
+ }
+ ],
+ "name":"DIR/modules/A",
+ "type":"directory"
+ }
+ ]
+}
+
+//--- build/unused-vfs.yaml.in
+
+{
+ "version":0,
+ "case-sensitive":"false",
+ "roots":[
+ {
+ "contents":[
+ {
+ "external-contents":"DIR/build/module.modulemap",
+ "name":"module.modulemap",
+ "type":"file"
+ }
+ ],
+ "name":"DIR/modules/D",
+ "type":"directory"
+ }
+ ]
+}
+
+//--- build/module.modulemap
+
+module A {
+ umbrella header "A.h"
+}
+
+//--- build/A.h
+
+typedef int A_t;
+
+//--- modules/B/module.modulemap
+
+module B {
+ umbrella header "B.h"
+}
+
+//--- modules/B/B.h
+
+typedef int B_t;
+
+//--- modules/C/module.modulemap
+
+module C {
+ umbrella header "C.h"
+}
+
+//--- modules/C/C.h
+
+ at import B;
+
+typedef B_t C_t;
+
+//--- 0.m
+
+#include <A.h>
+
+A_t a = 0;
+
+//--- A.m
+
+#include <A.h>
+
+A_t a = 0;
+
+//--- B.m
+
+#include <B.h>
+
+B_t b = 0;
+
+//--- C.m
+
+#include <C.h>
+
+C_t b = 0;
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index fb42dc1085afb39..76337664c45e382 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -156,6 +156,7 @@ static void ParseArgs(int argc, char **argv) {
.Case("none", ScanningOptimizations::None)
.Case("header-search", ScanningOptimizations::HeaderSearch)
.Case("system-warnings", ScanningOptimizations::SystemWarnings)
+ .Case("vfs", ScanningOptimizations::VFS)
.Case("all", ScanningOptimizations::All)
.Default(std::nullopt);
if (!Optimization) {
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 1c3d8e985592b67..1a5ea677db74daf 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -15,12 +15,14 @@
#define LLVM_SUPPORT_VIRTUALFILESYSTEM_H
#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/Support/Chrono.h"
-#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/ExtensibleRTTI.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/SourceMgr.h"
@@ -261,8 +263,10 @@ class recursive_directory_iterator {
};
/// The virtual file system interface.
-class FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem> {
+class FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem>,
+ public RTTIExtends<FileSystem, RTTIRoot> {
public:
+ static const char ID;
virtual ~FileSystem();
/// Get the status of the entry at \p Path, if one exists.
@@ -321,6 +325,13 @@ class FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem> {
printImpl(OS, Type, IndentLevel);
}
+ using VisitCallbackTy = llvm::function_ref<void(FileSystem &)>;
+ virtual void visitChildFileSystems(VisitCallbackTy Callback) {}
+ void visit(VisitCallbackTy Callback) {
+ Callback(*this);
+ visitChildFileSystems(Callback);
+ }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void dump() const;
#endif
@@ -360,7 +371,7 @@ std::unique_ptr<FileSystem> createPhysicalFileSystem();
/// top-most (most recently added) directory are used. When there is a file
/// that exists in more than one file system, the file in the top-most file
/// system overrides the other(s).
-class OverlayFileSystem : public FileSystem {
+class OverlayFileSystem : public RTTIExtends<OverlayFileSystem, FileSystem> {
using FileSystemList = SmallVector<IntrusiveRefCntPtr<FileSystem>, 1>;
/// The stack of file systems, implemented as a list in order of
@@ -368,6 +379,7 @@ class OverlayFileSystem : public FileSystem {
FileSystemList FSList;
public:
+ static const char ID;
OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> Base);
/// Pushes a file system on top of the stack.
@@ -412,13 +424,15 @@ class OverlayFileSystem : public FileSystem {
protected:
void printImpl(raw_ostream &OS, PrintType Type,
unsigned IndentLevel) const override;
+ void visitChildFileSystems(VisitCallbackTy Callback) override;
};
/// By default, this delegates all calls to the underlying file system. This
/// is useful when derived file systems want to override some calls and still
/// proxy other calls.
-class ProxyFileSystem : public FileSystem {
+class ProxyFileSystem : public RTTIExtends<ProxyFileSystem, FileSystem> {
public:
+ static const char ID;
explicit ProxyFileSystem(IntrusiveRefCntPtr<FileSystem> FS)
: FS(std::move(FS)) {}
@@ -448,11 +462,17 @@ class ProxyFileSystem : public FileSystem {
protected:
FileSystem &getUnderlyingFS() const { return *FS; }
+ void visitChildFileSystems(VisitCallbackTy Callback) override {
+ if (FS) {
+ Callback(*FS);
+ FS->visitChildFileSystems(Callback);
+ }
+ }
private:
IntrusiveRefCntPtr<FileSystem> FS;
- virtual void anchor();
+ virtual void anchor() override;
};
namespace detail {
@@ -495,11 +515,15 @@ class NamedNodeOrError {
} // namespace detail
/// An in-memory file system.
-class InMemoryFileSystem : public FileSystem {
+class InMemoryFileSystem : public RTTIExtends<InMemoryFileSystem, FileSystem> {
std::unique_ptr<detail::InMemoryDirectory> Root;
std::string WorkingDirectory;
bool UseNormalizedPaths = true;
+public:
+ static const char ID;
+
+private:
using MakeNodeFn = llvm::function_ref<std::unique_ptr<detail::InMemoryNode>(
detail::NewInMemoryNodeInfo)>;
@@ -736,8 +760,10 @@ class RedirectingFileSystemParser;
/// FIXME: 'use-external-name' causes behaviour that's inconsistent with how
/// "real" filesystems behave. Maybe there should be a separate channel for
/// this information.
-class RedirectingFileSystem : public vfs::FileSystem {
+class RedirectingFileSystem
+ : public RTTIExtends<RedirectingFileSystem, vfs::FileSystem> {
public:
+ static const char ID;
enum EntryKind { EK_Directory, EK_DirectoryRemap, EK_File };
enum NameKind { NK_NotSet, NK_External, NK_Virtual };
@@ -973,6 +999,13 @@ class RedirectingFileSystem : public vfs::FileSystem {
/// names of files. This global value is overridable on a per-file basis.
bool UseExternalNames = true;
+ /// True if this FS has redirected a lookup. This does not include
+ /// fallthrough.
+ mutable bool HasBeenUsed = false;
+
+ /// Used to enable or disable updating `HasBeenUsed`.
+ bool UsageTrackingActive = false;
+
/// Determines the lookups to perform, as well as their order. See
/// \c RedirectKind for details.
RedirectKind Redirection = RedirectKind::Fallthrough;
@@ -1043,11 +1076,17 @@ class RedirectingFileSystem : public vfs::FileSystem {
std::vector<llvm::StringRef> getRoots() const;
+ bool hasBeenUsed() const { return HasBeenUsed; };
+ void clearHasBeenUsed() { HasBeenUsed = false; }
+
+ void setUsageTrackingActive(bool Active) { UsageTrackingActive = Active; }
+
void printEntry(raw_ostream &OS, Entry *E, unsigned IndentLevel = 0) const;
protected:
void printImpl(raw_ostream &OS, PrintType Type,
unsigned IndentLevel) const override;
+ void visitChildFileSystems(VisitCallbackTy Callback) override;
};
/// Collect all pairs of <virtual path, real path> entries from the
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index d6c787cb91b3404..c34117068803748 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -480,6 +480,13 @@ OverlayFileSystem::getRealPath(const Twine &Path,
return errc::no_such_file_or_directory;
}
+void OverlayFileSystem::visitChildFileSystems(VisitCallbackTy Callback) {
+ for (IntrusiveRefCntPtr<FileSystem> FS : overlays_range()) {
+ Callback(*FS);
+ FS->visitChildFileSystems(Callback);
+ }
+}
+
void OverlayFileSystem::printImpl(raw_ostream &OS, PrintType Type,
unsigned IndentLevel) const {
printIndent(OS, IndentLevel);
@@ -1581,6 +1588,13 @@ void RedirectingFileSystem::printEntry(raw_ostream &OS,
}
}
+void RedirectingFileSystem::visitChildFileSystems(VisitCallbackTy Callback) {
+ if (ExternalFS) {
+ Callback(*ExternalFS);
+ ExternalFS->visitChildFileSystems(Callback);
+ }
+}
+
/// A helper class to hold the common YAML parsing state.
class llvm::vfs::RedirectingFileSystemParser {
yaml::Stream &Stream;
@@ -2263,12 +2277,18 @@ RedirectingFileSystem::makeCanonical(SmallVectorImpl<char> &Path) const {
ErrorOr<RedirectingFileSystem::LookupResult>
RedirectingFileSystem::lookupPath(StringRef Path) const {
+ // RedirectOnly means the VFS is always used.
+ if (UsageTrackingActive && Redirection == RedirectKind::RedirectOnly)
+ HasBeenUsed = true;
+
sys::path::const_iterator Start = sys::path::begin(Path);
sys::path::const_iterator End = sys::path::end(Path);
llvm::SmallVector<Entry *, 32> Entries;
for (const auto &Root : Roots) {
ErrorOr<RedirectingFileSystem::LookupResult> Result =
lookupPathImpl(Start, End, Root.get(), Entries);
+ if (UsageTrackingActive && Result && isa<RemapEntry>(Result->E))
+ HasBeenUsed = true;
if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) {
Result->Parents = std::move(Entries);
return Result;
@@ -2863,3 +2883,9 @@ recursive_directory_iterator::increment(std::error_code &EC) {
return *this;
}
+
+const char FileSystem::ID = 0;
+const char OverlayFileSystem::ID = 0;
+const char ProxyFileSystem::ID = 0;
+const char InMemoryFileSystem::ID = 0;
+const char RedirectingFileSystem::ID = 0;
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index 54a6e0fbc0760be..d4abbb4345873cf 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -895,6 +895,47 @@ TEST(VirtualFileSystemTest, HiddenInIteration) {
}
}
+TEST(VirtualFileSystemTest, Visit) {
+ IntrusiveRefCntPtr<DummyFileSystem> Base(new DummyFileSystem());
+ IntrusiveRefCntPtr<DummyFileSystem> Middle(new DummyFileSystem());
+ IntrusiveRefCntPtr<DummyFileSystem> Top(new DummyFileSystem());
+ IntrusiveRefCntPtr<vfs::OverlayFileSystem> O(
+ new vfs::OverlayFileSystem(Base));
+ O->pushOverlay(Middle);
+ O->pushOverlay(Top);
+
+ auto YAML =
+ MemoryBuffer::getMemBuffer("{\n"
+ " 'version': 0,\n"
+ " 'redirecting-with': 'redirect-only',\n"
+ " 'roots': [\n"
+ " {\n"
+ " 'type': 'file',\n"
+ " 'name': '/vfile',\n"
+ " 'external-contents': '/a',\n"
+ " },"
+ " ]\n"
+ "}");
+
+ IntrusiveRefCntPtr<vfs::RedirectingFileSystem> Redirecting =
+ vfs::RedirectingFileSystem::create(std::move(YAML), nullptr, "", nullptr,
+ O)
+ .release();
+
+ vfs::ProxyFileSystem PFS(Redirecting);
+
+ std::vector<const vfs::FileSystem *> FSs;
+ PFS.visit([&](const vfs::FileSystem &FS) { FSs.push_back(&FS); });
+
+ ASSERT_EQ(size_t(6), FSs.size());
+ EXPECT_TRUE(isa<vfs::ProxyFileSystem>(FSs[0]));
+ EXPECT_TRUE(isa<vfs::RedirectingFileSystem>(FSs[1]));
+ EXPECT_TRUE(isa<vfs::OverlayFileSystem>(FSs[2]));
+ EXPECT_TRUE(isa<vfs::FileSystem>(FSs[3]));
+ EXPECT_TRUE(isa<vfs::FileSystem>(FSs[4]));
+ EXPECT_TRUE(isa<vfs::FileSystem>(FSs[5]));
+}
+
TEST(OverlayFileSystemTest, PrintOutput) {
auto Dummy = makeIntrusiveRefCnt<DummyFileSystem>();
auto Overlay1 = makeIntrusiveRefCnt<vfs::OverlayFileSystem>(Dummy);
@@ -3244,3 +3285,48 @@ TEST(RedirectingFileSystemTest, PrintOutput) {
" DummyFileSystem (RecursiveContents)\n",
Output);
}
+
+TEST(RedirectingFileSystemTest, Used) {
+ auto Dummy = makeIntrusiveRefCnt<DummyFileSystem>();
+ auto YAML1 =
+ MemoryBuffer::getMemBuffer("{\n"
+ " 'version': 0,\n"
+ " 'redirecting-with': 'fallthrough',\n"
+ " 'roots': [\n"
+ " {\n"
+ " 'type': 'file',\n"
+ " 'name': '/vfile1',\n"
+ " 'external-contents': '/a',\n"
+ " },"
+ " ]\n"
+ "}");
+ auto YAML2 =
+ MemoryBuffer::getMemBuffer("{\n"
+ " 'version': 0,\n"
+ " 'redirecting-with': 'fallthrough',\n"
+ " 'roots': [\n"
+ " {\n"
+ " 'type': 'file',\n"
+ " 'name': '/vfile2',\n"
+ " 'external-contents': '/b',\n"
+ " },"
+ " ]\n"
+ "}");
+
+ Dummy->addRegularFile("/a");
+ Dummy->addRegularFile("/b");
+
+ IntrusiveRefCntPtr<vfs::RedirectingFileSystem> Redirecting1 =
+ vfs::RedirectingFileSystem::create(std::move(YAML1), nullptr, "", nullptr,
+ Dummy)
+ .release();
+ auto Redirecting2 = vfs::RedirectingFileSystem::create(
+ std::move(YAML2), nullptr, "", nullptr, Redirecting1);
+
+ Redirecting1->setUsageTrackingActive(true);
+ Redirecting2->setUsageTrackingActive(true);
+ EXPECT_TRUE(Redirecting2->exists("/vfile1"));
+ EXPECT_TRUE(Redirecting2->exists("/b"));
+ EXPECT_TRUE(Redirecting1->hasBeenUsed());
+ EXPECT_FALSE(Redirecting2->hasBeenUsed());
+}
More information about the cfe-commits
mailing list