[clang] e564fd9 - [clang][deps] Avoid minimizing PCH input files
Jan Svoboda via cfe-commits
cfe-commits at lists.llvm.org
Tue Jul 20 03:20:17 PDT 2021
Author: Jan Svoboda
Date: 2021-07-20T12:20:10+02:00
New Revision: e564fd93ab85d1810e0d83e961ccc8b99025ff34
URL: https://github.com/llvm/llvm-project/commit/e564fd93ab85d1810e0d83e961ccc8b99025ff34
DIFF: https://github.com/llvm/llvm-project/commit/e564fd93ab85d1810e0d83e961ccc8b99025ff34.diff
LOG: [clang][deps] Avoid minimizing PCH input files
This patch avoid minimizing input files that contributed to a PCH or its modules. This prevents the implicit modular build to fail on unexpected file size. Depends on D106146.
Reviewed By: dexonsmith
Differential Revision: https://reviews.llvm.org/D104536
Added:
Modified:
clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
clang/test/ClangScanDeps/modules-pch.c
Removed:
################################################################################
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 54581043c4f73..d651ff23b387a 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -46,25 +46,73 @@ class DependencyConsumerForwarder : public DependencyFileGenerator {
DependencyConsumer &C;
};
-/// A listener that collects the names and paths to imported modules.
-class ImportCollectingListener : public ASTReaderListener {
- using PrebuiltModuleFilesT =
- decltype(HeaderSearchOptions::PrebuiltModuleFiles);
-
+/// A listener that collects the imported modules and optionally the input
+/// files.
+class PrebuiltModuleListener : public ASTReaderListener {
public:
- ImportCollectingListener(PrebuiltModuleFilesT &PrebuiltModuleFiles)
- : PrebuiltModuleFiles(PrebuiltModuleFiles) {}
+ PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
+ llvm::StringSet<> &InputFiles, bool VisitInputFiles)
+ : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
+ VisitInputFiles(VisitInputFiles) {}
bool needsImportVisitation() const override { return true; }
+ bool needsInputFileVisitation() override { return VisitInputFiles; }
+ bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
void visitImport(StringRef ModuleName, StringRef Filename) override {
- PrebuiltModuleFiles[std::string(ModuleName)] = std::string(Filename);
+ PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
+ }
+
+ bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
+ bool isExplicitModule) override {
+ InputFiles.insert(Filename);
+ return true;
}
private:
- PrebuiltModuleFilesT &PrebuiltModuleFiles;
+ llvm::StringMap<std::string> &PrebuiltModuleFiles;
+ llvm::StringSet<> &InputFiles;
+ bool VisitInputFiles;
};
+using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
+
+/// Visit the given prebuilt module and collect all of the modules it
+/// transitively imports and contributing input files.
+static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
+ CompilerInstance &CI,
+ PrebuiltModuleFilesT &ModuleFiles,
+ llvm::StringSet<> &InputFiles,
+ bool VisitInputFiles) {
+ // Maps the names of modules that weren't yet visited to their PCM path.
+ llvm::StringMap<std::string> ModuleFilesWorklist;
+ // Contains PCM paths of all visited modules.
+ llvm::StringSet<> VisitedModuleFiles;
+
+ PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
+ VisitInputFiles);
+
+ auto GatherModuleFileInfo = [&](StringRef ASTFile) {
+ ASTReader::readASTFileControlBlock(
+ ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
+ /*FindModuleFileExtensions=*/false, Listener,
+ /*ValidateDiagnosticOptions=*/false);
+ };
+
+ GatherModuleFileInfo(PrebuiltModuleFilename);
+ while (!ModuleFilesWorklist.empty()) {
+ auto WorklistItemIt = ModuleFilesWorklist.begin();
+
+ if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
+ VisitedModuleFiles.insert(WorklistItemIt->getValue());
+ GatherModuleFileInfo(WorklistItemIt->getValue());
+ ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
+ }
+
+ ModuleFilesWorklist.erase(WorklistItemIt);
+ }
+}
+
/// Transform arbitrary file name into an object-like file name.
static std::string makeObjFileName(StringRef FileName) {
SmallString<128> ObjFileName(FileName);
@@ -122,13 +170,33 @@ class DependencyScanningAction : public tooling::ToolAction {
Compiler.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = true;
- // Use the dependency scanning optimized file system if we can.
+ FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
+ Compiler.setFileManager(FileMgr);
+ Compiler.createSourceManager(*FileMgr);
+
+ llvm::StringSet<> PrebuiltModulesInputFiles;
+ // Store the list of prebuilt module files into header search options. This
+ // will prevent the implicit build to create duplicate modules and will
+ // force reuse of the existing prebuilt module files instead.
+ if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty())
+ visitPrebuiltModule(
+ Compiler.getPreprocessorOpts().ImplicitPCHInclude, Compiler,
+ Compiler.getHeaderSearchOpts().PrebuiltModuleFiles,
+ PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
+
+ // Use the dependency scanning optimized file system if requested to do so.
if (DepFS) {
const CompilerInvocation &CI = Compiler.getInvocation();
+ DepFS->clearIgnoredFiles();
+ // Ignore any files that contributed to prebuilt modules. The implicit
+ // build validates the modules by comparing the reported sizes of their
+ // inputs to the current state of the filesystem. Minimization would throw
+ // this mechanism off.
+ for (const auto &File : PrebuiltModulesInputFiles)
+ DepFS->ignoreFile(File.getKey());
// Add any filenames that were explicity passed in the build settings and
// that might be opened, as we want to ensure we don't run source
// minimization on them.
- DepFS->clearIgnoredFiles();
for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries)
DepFS->ignoreFile(Entry.Path);
for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles)
@@ -146,24 +214,6 @@ class DependencyScanningAction : public tooling::ToolAction {
.ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
}
- FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
- Compiler.setFileManager(FileMgr);
- Compiler.createSourceManager(*FileMgr);
-
- if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
- // Collect the modules that were prebuilt as part of the PCH and pass them
- // to the compiler. This will prevent the implicit build to create
- // duplicate modules and force reuse of existing prebuilt module files
- // instead.
- ImportCollectingListener Listener(
- Compiler.getHeaderSearchOpts().PrebuiltModuleFiles);
- ASTReader::readASTFileControlBlock(
- Compiler.getPreprocessorOpts().ImplicitPCHInclude,
- Compiler.getFileManager(), Compiler.getPCHContainerReader(),
- /*FindModuleFileExtensions=*/false, Listener,
- /*ValidateDiagnosticOptions=*/false);
- }
-
// Create the dependency collector that will collect the produced
// dependencies.
//
diff --git a/clang/test/ClangScanDeps/modules-pch.c b/clang/test/ClangScanDeps/modules-pch.c
index 8b8b75310adb4..b9b7ed7badd10 100644
--- a/clang/test/ClangScanDeps/modules-pch.c
+++ b/clang/test/ClangScanDeps/modules-pch.c
@@ -6,7 +6,7 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb.json
// RUN: echo -%t > %t/result_pch.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN: -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_pch.json
+// RUN: -generate-modules-path-args -module-files-dir %t/build >> %t/result_pch.json
// RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-PCH
//
// Check we didn't build the PCH during dependency scanning.
@@ -127,9 +127,8 @@
//
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb.json
// RUN: echo -%t > %t/result_tu.json
-// FIXME: Make this work with '-mode preprocess-minimized-sources'.
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN: -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu.json
+// RUN: -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu.json
// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU
//
// CHECK-TU: -[[PREFIX:.*]]
@@ -193,7 +192,7 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu_with_common.json > %t/cdb.json
// RUN: echo -%t > %t/result_tu_with_common.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN: -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu_with_common.json
+// RUN: -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu_with_common.json
// RUN: cat %t/result_tu_with_common.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU-WITH-COMMON
//
// CHECK-TU-WITH-COMMON: -[[PREFIX:.*]]
More information about the cfe-commits
mailing list