[clang] [clang][DependencyScanning][NFC] Fold `CompilerInstaneWithContext` and `DependencyScannerImpl` (PR #208056)
Qiongsi Wu via cfe-commits
cfe-commits at lists.llvm.org
Tue Jul 7 10:53:44 PDT 2026
https://github.com/qiongsiwu created https://github.com/llvm/llvm-project/pull/208056
With https://github.com/llvm/llvm-project/pull/206800, we no longer need to expose `CompilerInstaneWithContext` or `DependencyScanningImpl` publicly. This PR folds the code in the header and source files into `DependencyScanningTool.cpp` and `DependencyScanningWorker.h/cpp`, and removes `CompilerInstaneWithContext.h/cpp` and `DependencyScannerImpl.h/cpp`.
>From 205d978fe9df9a9d89724cd1a138377c54c9c6f2 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Tue, 23 Jun 2026 13:51:23 -0700
Subject: [PATCH 01/13] First step to remove Driver dependency from
CompilerInstanceWithContext - standalong initializaion function.
---
.../clang/Tooling/DependencyScanningTool.h | 37 ++++++++--------
clang/lib/Tooling/DependencyScanningTool.cpp | 42 +++++++++++--------
2 files changed, 45 insertions(+), 34 deletions(-)
diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h
index 90216d8f1da82..6ff06cc573b32 100644
--- a/clang/include/clang/Tooling/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanningTool.h
@@ -117,6 +117,8 @@ class DependencyScanningTool {
return Worker.getTracingVFS();
}
+ dependencies::DependencyScanningWorker &getWorker() { return Worker; }
+
private:
dependencies::DependencyScanningWorker Worker;
@@ -172,9 +174,8 @@ class CompilerInstanceWithContext {
int32_t SrcLocOffset = 0;
CompilerInstanceWithContext(dependencies::DependencyScanningWorker &Worker,
- StringRef CWD,
- const std::vector<std::string> &CMD)
- : Worker(Worker), CWD(CWD), CommandLine(CMD) {};
+ StringRef CWD, ArrayRef<std::string> CMD)
+ : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {}
bool initialize(dependencies::DependencyActionController &Controller,
std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts>
@@ -182,21 +183,23 @@ class CompilerInstanceWithContext {
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS);
public:
- /// @brief Initialize the tool's compiler instance from the commandline.
- /// The compiler instance only takes a `-cc1` job, so this method
- /// builds the `-cc1` job from the CommandLine input.
- /// @param Tool The dependency scanning tool whose compiler instance
- /// with context is initialized.
+ /// @brief Initialize the tool's compiler instance from the cc1 commandline.
+ /// @param Worker The dependency scanning worker to initialize the compiler
+ /// instance.
/// @param CWD The current working directory.
- /// @param CommandLine This command line may be a driver command or a cc1
- /// command.
- /// @param DC A diagnostics consumer to report error if the initialization
- /// fails.
- static std::optional<CompilerInstanceWithContext> initializeFromCommandline(
- DependencyScanningTool &Tool, StringRef CWD,
- ArrayRef<std::string> CommandLine,
- dependencies::DependencyActionController &Controller,
- DiagnosticConsumer &DC);
+ /// @param CC1CommandLine A cc1 command.
+ /// @param DiagEngineWithDiagOpts The diagnostic engine used during scan.
+ /// @param OverlayFS An overlay FS containing the input file, which may be
+ /// from an in-memory buffer.
+ /// @param Controller A dependency action controller to gather some results.
+ static std::optional<CompilerInstanceWithContext>
+ initializeFromCC1Commandline(
+ dependencies::DependencyScanningWorker &Worker, StringRef CWD,
+ ArrayRef<std::string> CC1CommandLine,
+ std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts>
+ DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ dependencies::DependencyActionController &Controller);
/// @brief Initializing the context and the compiler instance.
/// This method must be called before calling
diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp
index 11b225830c2fc..ca8e76c833064 100644
--- a/clang/lib/Tooling/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanningTool.cpp
@@ -366,13 +366,13 @@ getFirstCC1CommandLine(ArrayRef<std::string> CommandLine,
return std::nullopt;
}
-std::optional<CompilerInstanceWithContext>
-CompilerInstanceWithContext::initializeFromCommandline(
+static std::optional<CompilerInstanceWithContext>
+createCompilerInstanceWithContextFromCommandline(
DependencyScanningTool &Tool, StringRef CWD,
ArrayRef<std::string> CommandLine, DependencyActionController &Controller,
DiagnosticConsumer &DC) {
auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(CommandLine);
- auto FS = Tool.Worker.makeEffectiveVFS(CWD, OverlayFS);
+ auto FS = Tool.getWorker().makeEffectiveVFS(CWD, OverlayFS);
auto DiagEngineWithCmdAndOpts =
std::make_unique<DiagnosticsEngineWithDiagOpts>(ModifiedCommandLine, FS,
@@ -381,13 +381,9 @@ CompilerInstanceWithContext::initializeFromCommandline(
if (ModifiedCommandLine.size() >= 2 && ModifiedCommandLine[1] == "-cc1") {
// The input command line is already a -cc1 invocation; initialize the
// compiler instance directly from it.
- CompilerInstanceWithContext CIWithContext(Tool.Worker, CWD,
- ModifiedCommandLine);
- if (!CIWithContext.initialize(Controller,
- std::move(DiagEngineWithCmdAndOpts),
- std::move(OverlayFS)))
- return std::nullopt;
- return std::move(CIWithContext);
+ return CompilerInstanceWithContext::initializeFromCC1Commandline(
+ Tool.getWorker(), CWD, ModifiedCommandLine,
+ std::move(DiagEngineWithCmdAndOpts), std::move(OverlayFS), Controller);
}
// The input command line is either a driver-style command line, or
@@ -400,12 +396,24 @@ CompilerInstanceWithContext::initializeFromCommandline(
std::vector<std::string> CC1CommandLine(MaybeFirstCC1->begin(),
MaybeFirstCC1->end());
- CompilerInstanceWithContext CIWithContext(Tool.Worker, CWD,
- std::move(CC1CommandLine));
- if (!CIWithContext.initialize(Controller, std::move(DiagEngineWithCmdAndOpts),
- std::move(OverlayFS)))
+ return CompilerInstanceWithContext::initializeFromCC1Commandline(
+ Tool.getWorker(), CWD, CC1CommandLine,
+ std::move(DiagEngineWithCmdAndOpts), std::move(OverlayFS), Controller);
+}
+
+std::optional<CompilerInstanceWithContext>
+CompilerInstanceWithContext::initializeFromCC1Commandline(
+ DependencyScanningWorker &Worker, StringRef CWD,
+ ArrayRef<std::string> CC1CommandLine,
+ std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts>
+ DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ DependencyActionController &Controller) {
+ CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine);
+ if (!CIWC.initialize(Controller, std::move(DiagEngineWithDiagOpts),
+ std::move(OverlayFS)))
return std::nullopt;
- return std::move(CIWithContext);
+ return std::move(CIWC);
}
llvm::Expected<CompilerInstanceWithContext>
@@ -415,8 +423,8 @@ CompilerInstanceWithContext::initializeOrError(
auto DiagPrinterWithOS =
std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine);
- auto Result = initializeFromCommandline(Tool, CWD, CommandLine, Controller,
- DiagPrinterWithOS->DiagPrinter);
+ auto Result = createCompilerInstanceWithContextFromCommandline(
+ Tool, CWD, CommandLine, Controller, DiagPrinterWithOS->DiagPrinter);
if (Result) {
Result->DiagPrinterWithOS = std::move(DiagPrinterWithOS);
return std::move(*Result);
>From 8986f184a639cc9be2dfb25a61aa2c1a3a27cc05 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Wed, 24 Jun 2026 09:23:42 -0700
Subject: [PATCH 02/13] Hide the CompilerInstanceWithContext (CIWC) from public
interfaces. It is an implementation detail that should not be exposed.
Preparation to move CIWC into its own header and cpp files.
---
.../clang/Tooling/DependencyScanningTool.h | 52 +++----------
clang/lib/Tooling/DependencyScanningTool.cpp | 74 ++++++++-----------
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 7 +-
3 files changed, 47 insertions(+), 86 deletions(-)
diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h
index 6ff06cc573b32..0a8355b5e923a 100644
--- a/clang/include/clang/Tooling/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanningTool.h
@@ -119,10 +119,20 @@ class DependencyScanningTool {
dependencies::DependencyScanningWorker &getWorker() { return Worker; }
+ llvm::Error initializeForByNameLookup(
+ StringRef CWD, ArrayRef<std::string> CommandLine,
+ dependencies::DependencyActionController &Controller);
+
+ llvm::Expected<dependencies::TranslationUnitDeps>
+ computeDependenciesByNameOrError(
+ StringRef ModuleName,
+ const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
+ dependencies::DependencyActionController &Controller);
+
private:
dependencies::DependencyScanningWorker Worker;
-
- friend class CompilerInstanceWithContext;
+ std::unique_ptr<dependencies::TextDiagnosticsPrinterWithOutput> DiagPrinter;
+ std::unique_ptr<CompilerInstanceWithContext> ByNameCIWC;
};
/// Run the dependency scanning worker for the given driver or frontend
@@ -150,13 +160,6 @@ class CompilerInstanceWithContext {
llvm::StringRef CWD;
std::vector<std::string> CommandLine;
- // Context - Diagnostics engine.
- DiagnosticConsumer *DiagConsumer = nullptr;
- std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts>
- DiagEngineWithCmdAndOpts;
- std::unique_ptr<dependencies::TextDiagnosticsPrinterWithOutput>
- DiagPrinterWithOS;
-
// Context - compiler invocation
std::unique_ptr<CompilerInvocation> OriginalInvocation;
@@ -201,42 +204,11 @@ class CompilerInstanceWithContext {
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
dependencies::DependencyActionController &Controller);
- /// @brief Initializing the context and the compiler instance.
- /// This method must be called before calling
- /// computeDependenciesByNameWithContext.
- /// @param CWD The current working directory used during the scan.
- /// @param CommandLine The commandline used for the scan.
- /// @return Error if the initializaiton fails.
- static llvm::Expected<CompilerInstanceWithContext>
- initializeOrError(DependencyScanningTool &Tool, StringRef CWD,
- ArrayRef<std::string> CommandLine,
- dependencies::DependencyActionController &Controller);
-
bool
computeDependencies(StringRef ModuleName,
dependencies::DependencyConsumer &Consumer,
dependencies::DependencyActionController &Controller);
- /// @brief Computes the dependeny for the module named ModuleName.
- /// @param ModuleName The name of the module for which this method computes
- ///. dependencies.
- /// @param AlreadySeen This stores modules which have previously been
- /// reported. Use the same instance for all calls to this
- /// function for a single \c DependencyScanningTool in a
- /// single build. Note that this parameter is not part of
- /// the context because it can be shared across different
- /// worker threads and each worker thread may update it.
- /// @param LookupModuleOutput This function is called to fill in
- /// "-fmodule-file=", "-o" and other output
- /// arguments for dependencies.
- /// @return An instance of \c TranslationUnitDeps if the scan is successful.
- /// Otherwise it returns an error.
- llvm::Expected<dependencies::TranslationUnitDeps>
- computeDependenciesByNameOrError(
- StringRef ModuleName,
- const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
- dependencies::DependencyActionController &Controller);
-
// MaxNumOfQueries is the upper limit of the number of names the by-name
// scanning API (computeDependencies) can support after a
// CompilerInstanceWithContext is initialized. At the time of this commit, the
diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp
index ca8e76c833064..1d5d950096490 100644
--- a/clang/lib/Tooling/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanningTool.cpp
@@ -334,13 +334,9 @@ DependencyScanningTool::getModuleDependencies(
StringRef ModuleName, ArrayRef<std::string> CommandLine, StringRef CWD,
const llvm::DenseSet<ModuleID> &AlreadySeen,
DependencyActionController &Controller) {
- auto MaybeCIWithContext = CompilerInstanceWithContext::initializeOrError(
- *this, CWD, CommandLine, Controller);
- if (auto Error = MaybeCIWithContext.takeError())
- return Error;
-
- return MaybeCIWithContext->computeDependenciesByNameOrError(
- ModuleName, AlreadySeen, Controller);
+ if (llvm::Error Err = initializeForByNameLookup(CWD, CommandLine, Controller))
+ return std::move(Err);
+ return computeDependenciesByNameOrError(ModuleName, AlreadySeen, Controller);
}
static std::optional<SmallVector<std::string, 0>>
@@ -401,6 +397,32 @@ createCompilerInstanceWithContextFromCommandline(
std::move(DiagEngineWithCmdAndOpts), std::move(OverlayFS), Controller);
}
+llvm::Error DependencyScanningTool::initializeForByNameLookup(
+ StringRef CWD, ArrayRef<std::string> CommandLine,
+ DependencyActionController &Controller) {
+ ByNameCIWC.reset();
+ DiagPrinter = std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine);
+ auto Result = createCompilerInstanceWithContextFromCommandline(
+ *this, CWD, CommandLine, Controller, DiagPrinter->DiagPrinter);
+ if (!Result)
+ return makeErrorFromDiagnosticsOS(*DiagPrinter);
+ ByNameCIWC =
+ std::make_unique<CompilerInstanceWithContext>(std::move(*Result));
+ return llvm::Error::success();
+}
+
+llvm::Expected<TranslationUnitDeps>
+DependencyScanningTool::computeDependenciesByNameOrError(
+ StringRef ModuleName, const llvm::DenseSet<ModuleID> &AlreadySeen,
+ DependencyActionController &Controller) {
+ assert(ByNameCIWC && "initializeForByNameLookup must be called first");
+ FullDependencyConsumer Consumer(AlreadySeen);
+ DiagPrinter->DiagnosticOutput.clear();
+ if (ByNameCIWC->computeDependencies(ModuleName, Consumer, Controller))
+ return Consumer.takeTranslationUnitDeps();
+ return makeErrorFromDiagnosticsOS(*DiagPrinter);
+}
+
std::optional<CompilerInstanceWithContext>
CompilerInstanceWithContext::initializeFromCC1Commandline(
DependencyScanningWorker &Worker, StringRef CWD,
@@ -416,50 +438,18 @@ CompilerInstanceWithContext::initializeFromCC1Commandline(
return std::move(CIWC);
}
-llvm::Expected<CompilerInstanceWithContext>
-CompilerInstanceWithContext::initializeOrError(
- DependencyScanningTool &Tool, StringRef CWD,
- ArrayRef<std::string> CommandLine, DependencyActionController &Controller) {
- auto DiagPrinterWithOS =
- std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine);
-
- auto Result = createCompilerInstanceWithContextFromCommandline(
- Tool, CWD, CommandLine, Controller, DiagPrinterWithOS->DiagPrinter);
- if (Result) {
- Result->DiagPrinterWithOS = std::move(DiagPrinterWithOS);
- return std::move(*Result);
- }
- return makeErrorFromDiagnosticsOS(*DiagPrinterWithOS);
-}
-
-llvm::Expected<TranslationUnitDeps>
-CompilerInstanceWithContext::computeDependenciesByNameOrError(
- StringRef ModuleName, const llvm::DenseSet<ModuleID> &AlreadySeen,
- DependencyActionController &Controller) {
- FullDependencyConsumer Consumer(AlreadySeen);
- // We need to clear the DiagnosticOutput so that each by-name lookup
- // has a clean diagnostics buffer.
- DiagPrinterWithOS->DiagnosticOutput.clear();
- if (computeDependencies(ModuleName, Consumer, Controller))
- return Consumer.takeTranslationUnitDeps();
- return makeErrorFromDiagnosticsOS(*DiagPrinterWithOS);
-}
-
bool CompilerInstanceWithContext::initialize(
DependencyActionController &Controller,
std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
- DiagEngineWithCmdAndOpts = std::move(DiagEngineWithDiagOpts);
- DiagConsumer = DiagEngineWithCmdAndOpts->DiagEngine->getClient();
-
assert(OverlayFS && "OverlayFS required!");
auto FS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
OriginalInvocation = createCompilerInvocation(
- CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine);
+ CommandLine, *DiagEngineWithDiagOpts->DiagEngine);
if (!OriginalInvocation) {
- DiagEngineWithCmdAndOpts->DiagEngine->Report(
+ DiagEngineWithDiagOpts->DiagEngine->Report(
diag::err_fe_expected_compiler_job)
<< llvm::join(CommandLine, " ");
return false;
@@ -479,7 +469,7 @@ bool CompilerInstanceWithContext::initialize(
auto &CI = *CIPtr;
initializeScanCompilerInstance(
- CI, std::move(FS), DiagEngineWithCmdAndOpts->DiagEngine->getClient(),
+ CI, std::move(FS), DiagEngineWithDiagOpts->DiagEngine->getClient(),
Worker.Service, Worker.DepFS);
StableDirs = getInitialStableDirs(CI);
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 1270f6f05a6c7..b586b1fd83111 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -1115,9 +1115,8 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
LocalIndex, DependencyOS, Errs))
HadErrors = true;
} else {
- auto CIWithCtx = CompilerInstanceWithContext::initializeOrError(
- WorkerTool, CWD, Input->CommandLine, Controller);
- if (llvm::Error Err = CIWithCtx.takeError()) {
+ if (llvm::Error Err = WorkerTool.initializeForByNameLookup(
+ CWD, Input->CommandLine, Controller)) {
handleErrorWithInfoString(
"Compiler instance with context setup error", std::move(Err),
DependencyOS, Errs);
@@ -1127,7 +1126,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
for (auto N : Names) {
auto MaybeModuleDepsGraph =
- CIWithCtx->computeDependenciesByNameOrError(
+ WorkerTool.computeDependenciesByNameOrError(
N, AlreadySeenModules, Controller);
if (handleModuleResult(N, MaybeModuleDepsGraph, *FD, LocalIndex,
DependencyOS, Errs)) {
>From f363c5119b7a2a0327ea02780757f246b36482ad Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Wed, 24 Jun 2026 11:15:51 -0700
Subject: [PATCH 03/13] Move CompilerInstaneWithContext to its own header and
implementation files.
---
.../CompilerInstanceWithContext.h | 81 +++++++
.../DependencyScanningWorker.h | 6 +-
.../clang/Tooling/DependencyScanningTool.h | 69 +-----
clang/lib/DependencyScanning/CMakeLists.txt | 1 +
.../CompilerInstanceWithContext.cpp | 193 +++++++++++++++++
clang/lib/Tooling/DependencyScanningTool.cpp | 197 ++----------------
6 files changed, 290 insertions(+), 257 deletions(-)
create mode 100644 clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
create mode 100644 clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
diff --git a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
new file mode 100644
index 0000000000000..192385a53174a
--- /dev/null
+++ b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_DEPENDENCYSCANNING_COMPILERINSTANCEWITHCONTEXT_H
+#define LLVM_CLANG_DEPENDENCYSCANNING_COMPILERINSTANCEWITHCONTEXT_H
+
+#include "clang/DependencyScanning/DependencyScanningWorker.h"
+
+namespace clang {
+namespace dependencies {
+class CompilerInstanceWithContext {
+ // Context
+ DependencyScanningWorker &Worker;
+ llvm::StringRef CWD;
+ std::vector<std::string> CommandLine;
+
+ // Context - compiler invocation
+ std::unique_ptr<CompilerInvocation> OriginalInvocation;
+
+ // Context - output options
+ std::unique_ptr<DependencyOutputOptions> OutputOpts;
+
+ // Context - stable directory handling
+ llvm::SmallVector<StringRef> StableDirs;
+ PrebuiltModulesAttrsMap PrebuiltModuleASTMap;
+
+ // Compiler Instance
+ std::unique_ptr<CompilerInstance> CIPtr;
+
+ // Source location offset.
+ int32_t SrcLocOffset = 0;
+
+ CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD,
+ ArrayRef<std::string> CMD)
+ : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {}
+
+ bool initialize(
+ DependencyActionController &Controller,
+ std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS);
+
+public:
+ /// @brief Initialize the tool's compiler instance from the cc1 commandline.
+ /// @param Worker The dependency scanning worker to initialize the compiler
+ /// instance.
+ /// @param CWD The current working directory.
+ /// @param CC1CommandLine A cc1 command.
+ /// @param DiagEngineWithDiagOpts The diagnostic engine used during scan.
+ /// @param OverlayFS An overlay FS containing the input file, which may be
+ /// from an in-memory buffer.
+ /// @param Controller A dependency action controller to gather some results.
+ static std::optional<CompilerInstanceWithContext>
+ initializeFromCC1Commandline(
+ DependencyScanningWorker &Worker, StringRef CWD,
+ ArrayRef<std::string> CC1CommandLine,
+ std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ DependencyActionController &Controller);
+
+ bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer,
+ DependencyActionController &Controller);
+
+ // MaxNumOfQueries is the upper limit of the number of names the by-name
+ // scanning API (computeDependencies) can support after a
+ // CompilerInstanceWithContext is initialized. At the time of this commit, the
+ // estimated number of total unique importable names is around 3000 from
+ // Apple's SDKs. We usually import them in parallel, so it is unlikely that
+ // all names are all scanned by the same dependency scanning worker. Therefore
+ // the 64k (20x bigger than our estimate) size is sufficient to hold the
+ // unique source locations to report diagnostics per worker.
+ static const int32_t MaxNumOfQueries = 1 << 16;
+};
+} // namespace dependencies
+} // namespace clang
+
+#endif
\ No newline at end of file
diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
index 1abcacf45fffd..37d7cd5524b71 100644
--- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
@@ -27,10 +27,6 @@ namespace clang {
class DependencyOutputOptions;
-namespace tooling {
-class CompilerInstanceWithContext;
-}
-
namespace dependencies {
class DependencyConsumer;
@@ -88,7 +84,7 @@ class DependencyScanningWorker {
/// The tracing VFS overlaid on top of the base VFS.
IntrusiveRefCntPtr<llvm::vfs::TracingFileSystem> TracingFS;
- friend tooling::CompilerInstanceWithContext;
+ friend class CompilerInstanceWithContext;
};
} // end namespace dependencies
} // end namespace clang
diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h
index 0a8355b5e923a..b21eb0705378f 100644
--- a/clang/include/clang/Tooling/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanningTool.h
@@ -9,6 +9,7 @@
#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
+#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/DependencyScanning/DependencyScanningService.h"
#include "clang/DependencyScanning/DependencyScanningUtils.h"
@@ -132,7 +133,7 @@ class DependencyScanningTool {
private:
dependencies::DependencyScanningWorker Worker;
std::unique_ptr<dependencies::TextDiagnosticsPrinterWithOutput> DiagPrinter;
- std::unique_ptr<CompilerInstanceWithContext> ByNameCIWC;
+ std::unique_ptr<dependencies::CompilerInstanceWithContext> ByNameCIWC;
};
/// Run the dependency scanning worker for the given driver or frontend
@@ -154,72 +155,6 @@ bool computeDependencies(
DiagnosticConsumer &DiagConsumer,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS = nullptr);
-class CompilerInstanceWithContext {
- // Context
- dependencies::DependencyScanningWorker &Worker;
- llvm::StringRef CWD;
- std::vector<std::string> CommandLine;
-
- // Context - compiler invocation
- std::unique_ptr<CompilerInvocation> OriginalInvocation;
-
- // Context - output options
- std::unique_ptr<DependencyOutputOptions> OutputOpts;
-
- // Context - stable directory handling
- llvm::SmallVector<StringRef> StableDirs;
- dependencies::PrebuiltModulesAttrsMap PrebuiltModuleASTMap;
-
- // Compiler Instance
- std::unique_ptr<CompilerInstance> CIPtr;
-
- // Source location offset.
- int32_t SrcLocOffset = 0;
-
- CompilerInstanceWithContext(dependencies::DependencyScanningWorker &Worker,
- StringRef CWD, ArrayRef<std::string> CMD)
- : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {}
-
- bool initialize(dependencies::DependencyActionController &Controller,
- std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts>
- DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS);
-
-public:
- /// @brief Initialize the tool's compiler instance from the cc1 commandline.
- /// @param Worker The dependency scanning worker to initialize the compiler
- /// instance.
- /// @param CWD The current working directory.
- /// @param CC1CommandLine A cc1 command.
- /// @param DiagEngineWithDiagOpts The diagnostic engine used during scan.
- /// @param OverlayFS An overlay FS containing the input file, which may be
- /// from an in-memory buffer.
- /// @param Controller A dependency action controller to gather some results.
- static std::optional<CompilerInstanceWithContext>
- initializeFromCC1Commandline(
- dependencies::DependencyScanningWorker &Worker, StringRef CWD,
- ArrayRef<std::string> CC1CommandLine,
- std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts>
- DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
- dependencies::DependencyActionController &Controller);
-
- bool
- computeDependencies(StringRef ModuleName,
- dependencies::DependencyConsumer &Consumer,
- dependencies::DependencyActionController &Controller);
-
- // MaxNumOfQueries is the upper limit of the number of names the by-name
- // scanning API (computeDependencies) can support after a
- // CompilerInstanceWithContext is initialized. At the time of this commit, the
- // estimated number of total unique importable names is around 3000 from
- // Apple's SDKs. We usually import them in parallel, so it is unlikely that
- // all names are all scanned by the same dependency scanning worker. Therefore
- // the 64k (20x bigger than our estimate) size is sufficient to hold the
- // unique source locations to report diagnostics per worker.
- static const int32_t MaxNumOfQueries = 1 << 16;
-};
-
} // end namespace tooling
} // end namespace clang
diff --git a/clang/lib/DependencyScanning/CMakeLists.txt b/clang/lib/DependencyScanning/CMakeLists.txt
index 015c6bcb12326..b7d85e6a09c02 100644
--- a/clang/lib/DependencyScanning/CMakeLists.txt
+++ b/clang/lib/DependencyScanning/CMakeLists.txt
@@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS
)
add_clang_library(clangDependencyScanning
+ CompilerInstanceWithContext.cpp
DependencyGraph.cpp
DependencyScanningFilesystem.cpp
DependencyScanningService.cpp
diff --git a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
new file mode 100644
index 0000000000000..c0d4bfa470908
--- /dev/null
+++ b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
@@ -0,0 +1,193 @@
+//===- CompilerInstanceWithContext.cpp - CI for dependency scanning -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticFrontend.h"
+#include "clang/DependencyScanning/DependencyActionController.h"
+#include "clang/DependencyScanning/DependencyConsumer.h"
+#include "clang/DependencyScanning/DependencyScannerImpl.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "llvm/ADT/ScopeExit.h"
+
+using namespace clang;
+using namespace dependencies;
+
+std::optional<CompilerInstanceWithContext>
+CompilerInstanceWithContext::initializeFromCC1Commandline(
+ DependencyScanningWorker &Worker, StringRef CWD,
+ ArrayRef<std::string> CC1CommandLine,
+ std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ DependencyActionController &Controller) {
+ CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine);
+ if (!CIWC.initialize(Controller, std::move(DiagEngineWithDiagOpts),
+ std::move(OverlayFS)))
+ return std::nullopt;
+ return std::move(CIWC);
+}
+
+bool CompilerInstanceWithContext::initialize(
+ DependencyActionController &Controller,
+ std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
+ assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
+ assert(OverlayFS && "OverlayFS required!");
+ auto FS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
+
+ OriginalInvocation = createCompilerInvocation(
+ CommandLine, *DiagEngineWithDiagOpts->DiagEngine);
+ if (!OriginalInvocation) {
+ DiagEngineWithDiagOpts->DiagEngine->Report(
+ diag::err_fe_expected_compiler_job)
+ << llvm::join(CommandLine, " ");
+ return false;
+ }
+
+ if (any(Worker.Service.getOpts().OptimizeArgs &
+ ScanningOptimizations::Macros))
+ canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
+
+ // Create the CompilerInstance.
+ std::shared_ptr<ModuleCache> ModCache =
+ makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());
+ CIPtr = std::make_unique<CompilerInstance>(
+ createScanCompilerInvocation(*OriginalInvocation, Worker.Service,
+ Controller),
+ Worker.PCHContainerOps, std::move(ModCache));
+ auto &CI = *CIPtr;
+
+ initializeScanCompilerInstance(
+ CI, std::move(FS), DiagEngineWithDiagOpts->DiagEngine->getClient(),
+ Worker.Service, Worker.DepFS);
+
+ StableDirs = getInitialStableDirs(CI);
+ auto MaybePrebuiltModulesASTMap =
+ computePrebuiltModulesASTMap(CI, StableDirs);
+ if (!MaybePrebuiltModulesASTMap)
+ return false;
+
+ PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
+ OutputOpts = createDependencyOutputOptions(*OriginalInvocation);
+
+ // We do not create the target in initializeScanCompilerInstance because
+ // setting it here is unique for by-name lookups. We create the target only
+ // once here, and the information is reused for all computeDependencies calls.
+ // We do not need to call createTarget explicitly if we go through
+ // CompilerInstance::ExecuteAction to perform scanning.
+ CI.createTarget();
+
+ return true;
+}
+
+bool CompilerInstanceWithContext::computeDependencies(
+ StringRef ModuleName, DependencyConsumer &Consumer,
+ DependencyActionController &Controller) {
+ if (SrcLocOffset >= MaxNumOfQueries)
+ llvm::report_fatal_error("exceeded maximum by-name scans for worker");
+
+ assert(CIPtr && "CIPtr must be initialized before calling this method");
+ auto &CI = *CIPtr;
+
+ // We need to reset the diagnostics, so that the diagnostics issued
+ // during a previous computeDependencies call do not affect the current call.
+ // If we do not reset, we may inherit fatal errors from a previous call.
+ CI.getDiagnostics().Reset();
+
+ // We create this cleanup object because computeDependencies may exit
+ // early with errors.
+ llvm::scope_exit CleanUp([&]() {
+ CI.clearDependencyCollectors();
+ // The preprocessor may not be created at the entry of this method,
+ // but it must have been created when this method returns, whether
+ // there are errors during scanning or not.
+ CI.getPreprocessor().removePPCallbacks();
+ });
+
+ auto MDC = initializeScanInstanceDependencyCollector(
+ CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
+ Worker.Service,
+ /* The MDC's constructor makes a copy of the OriginalInvocation, so
+ we can pass it in without worrying that it might be changed across
+ invocations of computeDependencies. */
+ *OriginalInvocation, Controller, PrebuiltModuleASTMap, StableDirs);
+
+ CompilerInvocation ModuleInvocation(*OriginalInvocation);
+ if (!Controller.initialize(CI, ModuleInvocation))
+ return false;
+
+ if (!SrcLocOffset) {
+ // When SrcLocOffset is zero, we are at the beginning of the fake source
+ // file. In this case, we call BeginSourceFile to initialize.
+ std::unique_ptr<FrontendAction> Action =
+ std::make_unique<PreprocessOnlyAction>();
+ auto *InputFile = CI.getFrontendOpts().Inputs.begin();
+ bool ActionBeginSucceeded = Action->BeginSourceFile(CI, *InputFile);
+ assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");
+ (void)ActionBeginSucceeded;
+ }
+
+ Preprocessor &PP = CI.getPreprocessor();
+ SourceManager &SM = PP.getSourceManager();
+ FileID MainFileID = SM.getMainFileID();
+ SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
+ SourceLocation IDLocation = FileStart.getLocWithOffset(SrcLocOffset);
+ PPCallbacks *CB = nullptr;
+ if (!SrcLocOffset) {
+ // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
+ // the preprocessor.
+ bool PPFailed = PP.EnterSourceFile(MainFileID, nullptr, SourceLocation());
+ assert(!PPFailed && "Preprocess must be able to enter the main file.");
+ (void)PPFailed;
+ CB = MDC->getPPCallbacks();
+ } else {
+ // When SrcLocOffset is non-zero, the preprocessor has already been
+ // initialized through a previous call of computeDependencies. We want to
+ // preserve the PP's state, hence we do not call EnterSourceFile again.
+ MDC->attachToPreprocessor(PP);
+ CB = MDC->getPPCallbacks();
+
+ FileID PrevFID;
+ SrcMgr::CharacteristicKind FileType = SM.getFileCharacteristic(IDLocation);
+ CB->LexedFileChanged(MainFileID,
+ PPChainedCallbacks::LexedFileChangeReason::EnterFile,
+ FileType, PrevFID, IDLocation);
+ }
+
+ // FIXME: Scan modules asynchronously here as well.
+
+ SrcLocOffset++;
+ SmallVector<IdentifierLoc, 2> Path;
+ IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
+ Path.emplace_back(IDLocation, ModuleID);
+ auto ModResult = CI.loadModule(IDLocation, Path, Module::Hidden, false);
+
+ assert(CB && "Must have PPCallbacks after module loading");
+ CB->moduleImport(SourceLocation(), Path, ModResult);
+
+ if (!ModResult)
+ return false;
+
+ if (CI.getDiagnostics().hasErrorOccurred())
+ return false;
+
+ MDC->run(Consumer);
+ MDC->applyDiscoveredDependencies(ModuleInvocation);
+
+ bool Success = ModuleInvocation.withCowRef<bool>(
+ [&](CowCompilerInvocation &CowModuleInvocation) {
+ return Controller.finalize(CI, CowModuleInvocation);
+ });
+ if (!Success)
+ return false;
+
+ Consumer.handleBuildCommand(
+ {CommandLine[0], ModuleInvocation.getCC1CommandLine()});
+
+ return true;
+}
diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp
index 1d5d950096490..b27396a739f36 100644
--- a/clang/lib/Tooling/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanningTool.cpp
@@ -286,7 +286,7 @@ initVFSForByNameScanning(ArrayRef<std::string> CommandLine) {
// locations for the diagnostics. Therefore, sharing this global buffer across
// threads is ok.
static const std::string FakeInput(
- CompilerInstanceWithContext::MaxNumOfQueries, ' ');
+ dependencies::CompilerInstanceWithContext::MaxNumOfQueries, ' ');
StringRef InputPath =
llvm::sys::path::is_style_windows(llvm::sys::path::Style::native)
@@ -362,7 +362,7 @@ getFirstCC1CommandLine(ArrayRef<std::string> CommandLine,
return std::nullopt;
}
-static std::optional<CompilerInstanceWithContext>
+static std::optional<dependencies::CompilerInstanceWithContext>
createCompilerInstanceWithContextFromCommandline(
DependencyScanningTool &Tool, StringRef CWD,
ArrayRef<std::string> CommandLine, DependencyActionController &Controller,
@@ -377,9 +377,10 @@ createCompilerInstanceWithContextFromCommandline(
if (ModifiedCommandLine.size() >= 2 && ModifiedCommandLine[1] == "-cc1") {
// The input command line is already a -cc1 invocation; initialize the
// compiler instance directly from it.
- return CompilerInstanceWithContext::initializeFromCC1Commandline(
- Tool.getWorker(), CWD, ModifiedCommandLine,
- std::move(DiagEngineWithCmdAndOpts), std::move(OverlayFS), Controller);
+ return dependencies::CompilerInstanceWithContext::
+ initializeFromCC1Commandline(Tool.getWorker(), CWD, ModifiedCommandLine,
+ std::move(DiagEngineWithCmdAndOpts),
+ std::move(OverlayFS), Controller);
}
// The input command line is either a driver-style command line, or
@@ -392,9 +393,10 @@ createCompilerInstanceWithContextFromCommandline(
std::vector<std::string> CC1CommandLine(MaybeFirstCC1->begin(),
MaybeFirstCC1->end());
- return CompilerInstanceWithContext::initializeFromCC1Commandline(
- Tool.getWorker(), CWD, CC1CommandLine,
- std::move(DiagEngineWithCmdAndOpts), std::move(OverlayFS), Controller);
+ return dependencies::CompilerInstanceWithContext::
+ initializeFromCC1Commandline(Tool.getWorker(), CWD, CC1CommandLine,
+ std::move(DiagEngineWithCmdAndOpts),
+ std::move(OverlayFS), Controller);
}
llvm::Error DependencyScanningTool::initializeForByNameLookup(
@@ -406,8 +408,8 @@ llvm::Error DependencyScanningTool::initializeForByNameLookup(
*this, CWD, CommandLine, Controller, DiagPrinter->DiagPrinter);
if (!Result)
return makeErrorFromDiagnosticsOS(*DiagPrinter);
- ByNameCIWC =
- std::make_unique<CompilerInstanceWithContext>(std::move(*Result));
+ ByNameCIWC = std::make_unique<dependencies::CompilerInstanceWithContext>(
+ std::move(*Result));
return llvm::Error::success();
}
@@ -422,178 +424,3 @@ DependencyScanningTool::computeDependenciesByNameOrError(
return Consumer.takeTranslationUnitDeps();
return makeErrorFromDiagnosticsOS(*DiagPrinter);
}
-
-std::optional<CompilerInstanceWithContext>
-CompilerInstanceWithContext::initializeFromCC1Commandline(
- DependencyScanningWorker &Worker, StringRef CWD,
- ArrayRef<std::string> CC1CommandLine,
- std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts>
- DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
- DependencyActionController &Controller) {
- CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine);
- if (!CIWC.initialize(Controller, std::move(DiagEngineWithDiagOpts),
- std::move(OverlayFS)))
- return std::nullopt;
- return std::move(CIWC);
-}
-
-bool CompilerInstanceWithContext::initialize(
- DependencyActionController &Controller,
- std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
- assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
- assert(OverlayFS && "OverlayFS required!");
- auto FS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
-
- OriginalInvocation = createCompilerInvocation(
- CommandLine, *DiagEngineWithDiagOpts->DiagEngine);
- if (!OriginalInvocation) {
- DiagEngineWithDiagOpts->DiagEngine->Report(
- diag::err_fe_expected_compiler_job)
- << llvm::join(CommandLine, " ");
- return false;
- }
-
- if (any(Worker.Service.getOpts().OptimizeArgs &
- ScanningOptimizations::Macros))
- canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
-
- // Create the CompilerInstance.
- std::shared_ptr<ModuleCache> ModCache =
- makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());
- CIPtr = std::make_unique<CompilerInstance>(
- createScanCompilerInvocation(*OriginalInvocation, Worker.Service,
- Controller),
- Worker.PCHContainerOps, std::move(ModCache));
- auto &CI = *CIPtr;
-
- initializeScanCompilerInstance(
- CI, std::move(FS), DiagEngineWithDiagOpts->DiagEngine->getClient(),
- Worker.Service, Worker.DepFS);
-
- StableDirs = getInitialStableDirs(CI);
- auto MaybePrebuiltModulesASTMap =
- computePrebuiltModulesASTMap(CI, StableDirs);
- if (!MaybePrebuiltModulesASTMap)
- return false;
-
- PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
- OutputOpts = createDependencyOutputOptions(*OriginalInvocation);
-
- // We do not create the target in initializeScanCompilerInstance because
- // setting it here is unique for by-name lookups. We create the target only
- // once here, and the information is reused for all computeDependencies calls.
- // We do not need to call createTarget explicitly if we go through
- // CompilerInstance::ExecuteAction to perform scanning.
- CI.createTarget();
-
- return true;
-}
-
-bool CompilerInstanceWithContext::computeDependencies(
- StringRef ModuleName, DependencyConsumer &Consumer,
- DependencyActionController &Controller) {
- if (SrcLocOffset >= MaxNumOfQueries)
- llvm::report_fatal_error("exceeded maximum by-name scans for worker");
-
- assert(CIPtr && "CIPtr must be initialized before calling this method");
- auto &CI = *CIPtr;
-
- // We need to reset the diagnostics, so that the diagnostics issued
- // during a previous computeDependencies call do not affect the current call.
- // If we do not reset, we may inherit fatal errors from a previous call.
- CI.getDiagnostics().Reset();
-
- // We create this cleanup object because computeDependencies may exit
- // early with errors.
- llvm::scope_exit CleanUp([&]() {
- CI.clearDependencyCollectors();
- // The preprocessor may not be created at the entry of this method,
- // but it must have been created when this method returns, whether
- // there are errors during scanning or not.
- CI.getPreprocessor().removePPCallbacks();
- });
-
- auto MDC = initializeScanInstanceDependencyCollector(
- CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
- Worker.Service,
- /* The MDC's constructor makes a copy of the OriginalInvocation, so
- we can pass it in without worrying that it might be changed across
- invocations of computeDependencies. */
- *OriginalInvocation, Controller, PrebuiltModuleASTMap, StableDirs);
-
- CompilerInvocation ModuleInvocation(*OriginalInvocation);
- if (!Controller.initialize(CI, ModuleInvocation))
- return false;
-
- if (!SrcLocOffset) {
- // When SrcLocOffset is zero, we are at the beginning of the fake source
- // file. In this case, we call BeginSourceFile to initialize.
- std::unique_ptr<FrontendAction> Action =
- std::make_unique<PreprocessOnlyAction>();
- auto *InputFile = CI.getFrontendOpts().Inputs.begin();
- bool ActionBeginSucceeded = Action->BeginSourceFile(CI, *InputFile);
- assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");
- (void)ActionBeginSucceeded;
- }
-
- Preprocessor &PP = CI.getPreprocessor();
- SourceManager &SM = PP.getSourceManager();
- FileID MainFileID = SM.getMainFileID();
- SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
- SourceLocation IDLocation = FileStart.getLocWithOffset(SrcLocOffset);
- PPCallbacks *CB = nullptr;
- if (!SrcLocOffset) {
- // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
- // the preprocessor.
- bool PPFailed = PP.EnterSourceFile(MainFileID, nullptr, SourceLocation());
- assert(!PPFailed && "Preprocess must be able to enter the main file.");
- (void)PPFailed;
- CB = MDC->getPPCallbacks();
- } else {
- // When SrcLocOffset is non-zero, the preprocessor has already been
- // initialized through a previous call of computeDependencies. We want to
- // preserve the PP's state, hence we do not call EnterSourceFile again.
- MDC->attachToPreprocessor(PP);
- CB = MDC->getPPCallbacks();
-
- FileID PrevFID;
- SrcMgr::CharacteristicKind FileType = SM.getFileCharacteristic(IDLocation);
- CB->LexedFileChanged(MainFileID,
- PPChainedCallbacks::LexedFileChangeReason::EnterFile,
- FileType, PrevFID, IDLocation);
- }
-
- // FIXME: Scan modules asynchronously here as well.
-
- SrcLocOffset++;
- SmallVector<IdentifierLoc, 2> Path;
- IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
- Path.emplace_back(IDLocation, ModuleID);
- auto ModResult = CI.loadModule(IDLocation, Path, Module::Hidden, false);
-
- assert(CB && "Must have PPCallbacks after module loading");
- CB->moduleImport(SourceLocation(), Path, ModResult);
-
- if (!ModResult)
- return false;
-
- if (CI.getDiagnostics().hasErrorOccurred())
- return false;
-
- MDC->run(Consumer);
- MDC->applyDiscoveredDependencies(ModuleInvocation);
-
- bool Success = ModuleInvocation.withCowRef<bool>(
- [&](CowCompilerInvocation &CowModuleInvocation) {
- return Controller.finalize(CI, CowModuleInvocation);
- });
- if (!Success)
- return false;
-
- Consumer.handleBuildCommand(
- {CommandLine[0], ModuleInvocation.getCC1CommandLine()});
-
- return true;
-}
>From 2dca9dfd1e564d09b8d70cad6be5588664e8df24 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Wed, 24 Jun 2026 15:19:36 -0700
Subject: [PATCH 04/13] 1. Pull out a CIWC method to collect scanning results.
---
.../CompilerInstanceWithContext.h | 6 ++++
.../CompilerInstanceWithContext.cpp | 33 +++++++++++--------
2 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
index 192385a53174a..1931c2f38ee3c 100644
--- a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
+++ b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
@@ -44,6 +44,12 @@ class CompilerInstanceWithContext {
std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS);
+ bool applyAndReport(ModuleDepCollector &MDC,
+ CompilerInvocation &ModuleInvocation,
+ DependencyConsumer &Consumer,
+ DependencyActionController &Controller,
+ StringRef Executable);
+
public:
/// @brief Initialize the tool's compiler instance from the cc1 commandline.
/// @param Worker The dependency scanning worker to initialize the compiler
diff --git a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
index c0d4bfa470908..43ef0cc48d2ef 100644
--- a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
+++ b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
@@ -85,6 +85,24 @@ bool CompilerInstanceWithContext::initialize(
return true;
}
+bool CompilerInstanceWithContext::applyAndReport(
+ ModuleDepCollector &MDC, CompilerInvocation &ModuleInvocation,
+ DependencyConsumer &Consumer, DependencyActionController &Controller,
+ StringRef Executable) {
+ MDC.applyDiscoveredDependencies(ModuleInvocation);
+
+ bool Success = ModuleInvocation.withCowRef<bool>(
+ [&](CowCompilerInvocation &CowModuleInvocation) {
+ return Controller.finalize(*CIPtr, CowModuleInvocation);
+ });
+ if (!Success)
+ return false;
+
+ Consumer.handleBuildCommand(
+ {Executable.str(), ModuleInvocation.getCC1CommandLine()});
+ return true;
+}
+
bool CompilerInstanceWithContext::computeDependencies(
StringRef ModuleName, DependencyConsumer &Consumer,
DependencyActionController &Controller) {
@@ -177,17 +195,6 @@ bool CompilerInstanceWithContext::computeDependencies(
return false;
MDC->run(Consumer);
- MDC->applyDiscoveredDependencies(ModuleInvocation);
-
- bool Success = ModuleInvocation.withCowRef<bool>(
- [&](CowCompilerInvocation &CowModuleInvocation) {
- return Controller.finalize(CI, CowModuleInvocation);
- });
- if (!Success)
- return false;
-
- Consumer.handleBuildCommand(
- {CommandLine[0], ModuleInvocation.getCC1CommandLine()});
-
- return true;
+ return applyAndReport(*MDC, ModuleInvocation, Consumer, Controller,
+ CommandLine[0]);
}
>From 8833de75290ae6fe57ad18d30cec9033b7741692 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Thu, 25 Jun 2026 10:33:54 -0700
Subject: [PATCH 05/13] 2-pre. Sink the CIWC into the dependency scanning
worker.
---
.../DependencyScanningWorker.h | 15 +++++++
.../clang/Tooling/DependencyScanningTool.h | 6 +--
.../DependencyScanningWorker.cpp | 25 ++++++++++++
clang/lib/Tooling/DependencyScanningTool.cpp | 39 ++++++++-----------
4 files changed, 60 insertions(+), 25 deletions(-)
diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
index 37d7cd5524b71..428ef43a83044 100644
--- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
@@ -31,6 +31,7 @@ namespace dependencies {
class DependencyConsumer;
class DependencyScanningWorkerFilesystem;
+class CompilerInstanceWithContext;
/// An individual dependency scanning worker that is able to run on its own
/// thread.
@@ -74,6 +75,18 @@ class DependencyScanningWorker {
return TracingFS.get();
}
+ bool initializeCIWC(
+ StringRef CWD, ArrayRef<std::string> CC1CommandLine,
+ std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ DependencyActionController &Controller);
+
+ void resetCIWC();
+
+ bool computeDependenciesByName(StringRef ModuleName,
+ DependencyConsumer &Consumer,
+ DependencyActionController &Controller);
+
private:
/// The parent dependency scanning service.
DependencyScanningService &Service;
@@ -84,6 +97,8 @@ class DependencyScanningWorker {
/// The tracing VFS overlaid on top of the base VFS.
IntrusiveRefCntPtr<llvm::vfs::TracingFileSystem> TracingFS;
+ std::unique_ptr<CompilerInstanceWithContext> CIWC;
+
friend class CompilerInstanceWithContext;
};
} // end namespace dependencies
diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h
index b21eb0705378f..9bda50ee858f9 100644
--- a/clang/include/clang/Tooling/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanningTool.h
@@ -9,7 +9,6 @@
#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
-#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/DependencyScanning/DependencyScanningService.h"
#include "clang/DependencyScanning/DependencyScanningUtils.h"
@@ -131,9 +130,10 @@ class DependencyScanningTool {
dependencies::DependencyActionController &Controller);
private:
- dependencies::DependencyScanningWorker Worker;
+ // Dependency scanning worker has components that depends on the DiagPrinter.
+ // Hence the DiagPrinter is declared first. Do not change the ordering.
std::unique_ptr<dependencies::TextDiagnosticsPrinterWithOutput> DiagPrinter;
- std::unique_ptr<dependencies::CompilerInstanceWithContext> ByNameCIWC;
+ dependencies::DependencyScanningWorker Worker;
};
/// Run the dependency scanning worker for the given driver or frontend
diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
index 166a61bc4933c..7b19929eeb20f 100644
--- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
@@ -9,6 +9,7 @@
#include "clang/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticFrontend.h"
+#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
#include "clang/DependencyScanning/DependencyConsumer.h"
#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
@@ -71,6 +72,30 @@ DependencyScanningWorker::makeEffectiveVFS(
return FS;
}
+bool DependencyScanningWorker::initializeCIWC(
+ StringRef CWD, ArrayRef<std::string> CC1CommandLine,
+ std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ DependencyActionController &Controller) {
+ CIWC.reset();
+ auto Result = CompilerInstanceWithContext::initializeFromCC1Commandline(
+ *this, CWD, CC1CommandLine, std::move(DiagEngineWithDiagOpts),
+ std::move(OverlayFS), Controller);
+ if (!Result)
+ return false;
+ CIWC = std::make_unique<CompilerInstanceWithContext>(std::move(*Result));
+ return true;
+}
+
+void DependencyScanningWorker::resetCIWC() { CIWC.reset(); }
+
+bool DependencyScanningWorker::computeDependenciesByName(
+ StringRef ModuleName, DependencyConsumer &Consumer,
+ DependencyActionController &Controller) {
+ assert(CIWC && "initializeCIWC must succeed before calling this method");
+ return CIWC->computeDependencies(ModuleName, Consumer, Controller);
+}
+
bool DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines,
DependencyConsumer &DepConsumer, DependencyActionController &Controller,
diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp
index b27396a739f36..2432d20ff24fa 100644
--- a/clang/lib/Tooling/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanningTool.cpp
@@ -9,6 +9,7 @@
#include "clang/Tooling/DependencyScanningTool.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticFrontend.h"
+#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
@@ -362,11 +363,11 @@ getFirstCC1CommandLine(ArrayRef<std::string> CommandLine,
return std::nullopt;
}
-static std::optional<dependencies::CompilerInstanceWithContext>
-createCompilerInstanceWithContextFromCommandline(
- DependencyScanningTool &Tool, StringRef CWD,
- ArrayRef<std::string> CommandLine, DependencyActionController &Controller,
- DiagnosticConsumer &DC) {
+static bool
+initializeWorkerForByNameLookup(DependencyScanningTool &Tool, StringRef CWD,
+ ArrayRef<std::string> CommandLine,
+ DependencyActionController &Controller,
+ DiagnosticConsumer &DC) {
auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(CommandLine);
auto FS = Tool.getWorker().makeEffectiveVFS(CWD, OverlayFS);
@@ -377,10 +378,9 @@ createCompilerInstanceWithContextFromCommandline(
if (ModifiedCommandLine.size() >= 2 && ModifiedCommandLine[1] == "-cc1") {
// The input command line is already a -cc1 invocation; initialize the
// compiler instance directly from it.
- return dependencies::CompilerInstanceWithContext::
- initializeFromCC1Commandline(Tool.getWorker(), CWD, ModifiedCommandLine,
- std::move(DiagEngineWithCmdAndOpts),
- std::move(OverlayFS), Controller);
+ return Tool.getWorker().initializeCIWC(CWD, ModifiedCommandLine,
+ std::move(DiagEngineWithCmdAndOpts),
+ std::move(OverlayFS), Controller);
}
// The input command line is either a driver-style command line, or
@@ -389,27 +389,23 @@ createCompilerInstanceWithContextFromCommandline(
const auto MaybeFirstCC1 = getFirstCC1CommandLine(
ModifiedCommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, FS);
if (!MaybeFirstCC1)
- return std::nullopt;
+ return false;
std::vector<std::string> CC1CommandLine(MaybeFirstCC1->begin(),
MaybeFirstCC1->end());
- return dependencies::CompilerInstanceWithContext::
- initializeFromCC1Commandline(Tool.getWorker(), CWD, CC1CommandLine,
- std::move(DiagEngineWithCmdAndOpts),
- std::move(OverlayFS), Controller);
+ return Tool.getWorker().initializeCIWC(CWD, CC1CommandLine,
+ std::move(DiagEngineWithCmdAndOpts),
+ std::move(OverlayFS), Controller);
}
llvm::Error DependencyScanningTool::initializeForByNameLookup(
StringRef CWD, ArrayRef<std::string> CommandLine,
DependencyActionController &Controller) {
- ByNameCIWC.reset();
+ Worker.resetCIWC();
DiagPrinter = std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine);
- auto Result = createCompilerInstanceWithContextFromCommandline(
- *this, CWD, CommandLine, Controller, DiagPrinter->DiagPrinter);
- if (!Result)
+ if (!initializeWorkerForByNameLookup(*this, CWD, CommandLine, Controller,
+ DiagPrinter->DiagPrinter))
return makeErrorFromDiagnosticsOS(*DiagPrinter);
- ByNameCIWC = std::make_unique<dependencies::CompilerInstanceWithContext>(
- std::move(*Result));
return llvm::Error::success();
}
@@ -417,10 +413,9 @@ llvm::Expected<TranslationUnitDeps>
DependencyScanningTool::computeDependenciesByNameOrError(
StringRef ModuleName, const llvm::DenseSet<ModuleID> &AlreadySeen,
DependencyActionController &Controller) {
- assert(ByNameCIWC && "initializeForByNameLookup must be called first");
FullDependencyConsumer Consumer(AlreadySeen);
DiagPrinter->DiagnosticOutput.clear();
- if (ByNameCIWC->computeDependencies(ModuleName, Consumer, Controller))
+ if (Worker.computeDependenciesByName(ModuleName, Consumer, Controller))
return Consumer.takeTranslationUnitDeps();
return makeErrorFromDiagnosticsOS(*DiagPrinter);
}
>From 7e09c16b1d27e06d8474e09658c240d32dcfd72e Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Thu, 25 Jun 2026 14:47:28 -0700
Subject: [PATCH 06/13] 2. Use the CIWC to drive TU scanning as well.
---
.../CompilerInstanceWithContext.h | 16 ++++---
.../CompilerInstanceWithContext.cpp | 29 +++++++++++-
.../DependencyScanningWorker.cpp | 46 +++++++++----------
3 files changed, 61 insertions(+), 30 deletions(-)
diff --git a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
index 1931c2f38ee3c..ff59d0c394a39 100644
--- a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
+++ b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
@@ -44,12 +44,6 @@ class CompilerInstanceWithContext {
std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS);
- bool applyAndReport(ModuleDepCollector &MDC,
- CompilerInvocation &ModuleInvocation,
- DependencyConsumer &Consumer,
- DependencyActionController &Controller,
- StringRef Executable);
-
public:
/// @brief Initialize the tool's compiler instance from the cc1 commandline.
/// @param Worker The dependency scanning worker to initialize the compiler
@@ -80,6 +74,16 @@ class CompilerInstanceWithContext {
// the 64k (20x bigger than our estimate) size is sufficient to hold the
// unique source locations to report diagnostics per worker.
static const int32_t MaxNumOfQueries = 1 << 16;
+
+ std::shared_ptr<ModuleDepCollector>
+ scanTranslationUnit(DependencyConsumer &Consumer,
+ DependencyActionController &Controller);
+
+ bool applyAndReport(ModuleDepCollector &MDC,
+ CompilerInvocation &ModuleInvocation,
+ DependencyConsumer &Consumer,
+ DependencyActionController &Controller,
+ StringRef Executable);
};
} // namespace dependencies
} // namespace clang
diff --git a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
index 43ef0cc48d2ef..868b79cc004bd 100644
--- a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
+++ b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
@@ -37,7 +37,6 @@ bool CompilerInstanceWithContext::initialize(
std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
- assert(OverlayFS && "OverlayFS required!");
auto FS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
OriginalInvocation = createCompilerInvocation(
@@ -198,3 +197,31 @@ bool CompilerInstanceWithContext::computeDependencies(
return applyAndReport(*MDC, ModuleInvocation, Consumer, Controller,
CommandLine[0]);
}
+
+std::shared_ptr<ModuleDepCollector>
+CompilerInstanceWithContext::scanTranslationUnit(
+ DependencyConsumer &Consumer, DependencyActionController &Controller) {
+ assert(CIPtr && "CIPtr must be initialized before calling this method");
+ auto &CI = *CIPtr;
+
+ auto MDC = initializeScanInstanceDependencyCollector(
+ CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
+ Worker.Service, *OriginalInvocation, Controller, PrebuiltModuleASTMap,
+ StableDirs);
+
+ if (CI.getDiagnostics().hasErrorOccurred())
+ return nullptr;
+
+ if (!Controller.initialize(CI, *OriginalInvocation))
+ return nullptr;
+
+ ReadPCHAndPreprocessAction Action;
+ if (!CI.ExecuteAction(Action))
+ return nullptr;
+
+ MDC->run(Consumer);
+ if (!applyAndReport(*MDC, *OriginalInvocation, Consumer, Controller,
+ CommandLine[0]))
+ return nullptr;
+ return MDC;
+}
diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
index 7b19929eeb20f..999bba284ff39 100644
--- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
@@ -43,20 +43,6 @@ DependencyScanningWorker::DependencyScanningWorker(
DependencyScanningWorker::~DependencyScanningWorker() = default;
-static bool createAndRunToolInvocation(
- ArrayRef<std::string> CommandLine, DependencyScanningAction &Action,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- std::shared_ptr<clang::PCHContainerOperations> &PCHContainerOps,
- DiagnosticsEngine &Diags) {
- auto Invocation = createCompilerInvocation(CommandLine, Diags);
- if (!Invocation)
- return false;
-
- return Action.runInvocation(CommandLine[0], std::move(Invocation),
- std::move(FS), PCHContainerOps,
- Diags.getClient());
-}
-
IntrusiveRefCntPtr<llvm::vfs::FileSystem>
DependencyScanningWorker::makeEffectiveVFS(
StringRef WorkingDirectory,
@@ -103,8 +89,8 @@ bool DependencyScanningWorker::computeDependencies(
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
auto FS = makeEffectiveVFS(WorkingDirectory, std::move(OverlayFS));
- DependencyScanningAction Action(Service, WorkingDirectory, DepConsumer,
- Controller, DepFS);
+ bool Scanned = false;
+ std::shared_ptr<ModuleDepCollector> MDC;
const bool Success = llvm::all_of(CommandLines, [&](const auto &Cmd) {
if (StringRef(Cmd[1]) != "-cc1") {
@@ -115,14 +101,28 @@ bool DependencyScanningWorker::computeDependencies(
}
auto DiagEngineWithDiagOpts =
- DiagnosticsEngineWithDiagOpts(Cmd, FS, DiagConsumer);
- auto &Diags = *DiagEngineWithDiagOpts.DiagEngine;
+ std::make_unique<DiagnosticsEngineWithDiagOpts>(Cmd, FS, DiagConsumer);
+ if (!Scanned) {
+ Scanned = true;
+ if (!initializeCIWC(WorkingDirectory, Cmd,
+ std::move(DiagEngineWithDiagOpts), OverlayFS,
+ Controller))
+ return false;
+ MDC = CIWC->scanTranslationUnit(DepConsumer, Controller);
+ return MDC != nullptr;
+ }
+
+ auto Invocation =
+ createCompilerInvocation(Cmd, *DiagEngineWithDiagOpts->DiagEngine);
+
+ if (!Invocation)
+ return false;
- // Create an invocation that uses the underlying file system to ensure that
- // any file system requests that are made by the driver do not go through
- // the dependency scanning filesystem.
- return createAndRunToolInvocation(Cmd, Action, FS, PCHContainerOps, Diags);
+ if (!Invocation)
+ return false;
+ return CIWC->applyAndReport(*MDC, *Invocation, DepConsumer, Controller,
+ Cmd.front());
});
- return Success && Action.hasScanned();
+ return Success && Scanned;
}
>From 96f330822fa616983033aa9f7474720f99b5b7a0 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Fri, 26 Jun 2026 13:40:27 -0700
Subject: [PATCH 07/13] 3. Move async scanning logic into
CompilerInstanceWithContext.
---
.../CompilerInstanceWithContext.h | 6 ++
.../DependencyScannerImpl.h | 37 +++++++++++
.../CompilerInstanceWithContext.cpp | 39 ++++++++++-
.../DependencyScannerImpl.cpp | 37 +++--------
clang/test/ClangScanDeps/modules-async-scan.c | 65 +++++++++++++++++++
5 files changed, 153 insertions(+), 31 deletions(-)
create mode 100644 clang/test/ClangScanDeps/modules-async-scan.c
diff --git a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
index ff59d0c394a39..9d93fbf15ee62 100644
--- a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
+++ b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
@@ -29,6 +29,9 @@ class CompilerInstanceWithContext {
llvm::SmallVector<StringRef> StableDirs;
PrebuiltModulesAttrsMap PrebuiltModuleASTMap;
+ // Context - used by AsyncScan's prescan pass
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS;
+
// Compiler Instance
std::unique_ptr<CompilerInstance> CIPtr;
@@ -44,6 +47,9 @@ class CompilerInstanceWithContext {
std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS);
+ bool prescanModulesAsync(AsyncModuleCompiles &Compiles,
+ DependencyActionController &Controller);
+
public:
/// @brief Initialize the tool's compiler instance from the cc1 commandline.
/// @param Worker The dependency scanning worker to initialize the compiler
diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
index 893017e68233b..4507400d2da8c 100644
--- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
+++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
@@ -16,6 +16,9 @@
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "llvm/Support/VirtualFileSystem.h"
+#include <mutex>
+#include <thread>
+
namespace clang {
class DiagnosticConsumer;
@@ -83,6 +86,40 @@ struct TextDiagnosticsPrinterWithOutput {
DiagPrinter(DiagnosticsOS, *DiagOpts) {}
};
+/// Manages (and terminates) the asynchronous compilation of modules.
+class AsyncModuleCompiles {
+ std::mutex Mutex;
+ bool Stop = false;
+ // FIXME: Have the service own a thread pool and use that instead.
+ std::vector<std::thread> Compiles;
+
+public:
+ /// Registers the module compilation, unless this instance is about to be
+ /// destroyed.
+ void add(llvm::unique_function<void()> Compile) {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ if (!Stop)
+ Compiles.emplace_back(std::move(Compile));
+ }
+
+ ~AsyncModuleCompiles() {
+ {
+ // Prevent registration of further module compiles.
+ std::lock_guard<std::mutex> Lock(Mutex);
+ Stop = true;
+ }
+
+ // Wait for outstanding module compiles to finish.
+ for (std::thread &Compile : Compiles)
+ Compile.join();
+ }
+};
+
+void runTUModulePrescan(CompilerInstance &PrescanCI,
+ DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles);
+
std::unique_ptr<CompilerInvocation>
createCompilerInvocation(ArrayRef<std::string> CommandLine,
DiagnosticsEngine &Diags);
diff --git a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
index 868b79cc004bd..bd2139329df1a 100644
--- a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
+++ b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
@@ -12,8 +12,12 @@
#include "clang/DependencyScanning/DependencyActionController.h"
#include "clang/DependencyScanning/DependencyConsumer.h"
#include "clang/DependencyScanning/DependencyScannerImpl.h"
+#include "clang/DependencyScanning/InProcessModuleCache.h"
+#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/FrontendOptions.h"
#include "llvm/ADT/ScopeExit.h"
+#include <optional>
using namespace clang;
using namespace dependencies;
@@ -37,7 +41,7 @@ bool CompilerInstanceWithContext::initialize(
std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
- auto FS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
+ ScanFS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
OriginalInvocation = createCompilerInvocation(
CommandLine, *DiagEngineWithDiagOpts->DiagEngine);
@@ -62,7 +66,7 @@ bool CompilerInstanceWithContext::initialize(
auto &CI = *CIPtr;
initializeScanCompilerInstance(
- CI, std::move(FS), DiagEngineWithDiagOpts->DiagEngine->getClient(),
+ CI, ScanFS, DiagEngineWithDiagOpts->DiagEngine->getClient(),
Worker.Service, Worker.DepFS);
StableDirs = getInitialStableDirs(CI);
@@ -84,6 +88,30 @@ bool CompilerInstanceWithContext::initialize(
return true;
}
+bool CompilerInstanceWithContext::prescanModulesAsync(
+ AsyncModuleCompiles &Compiles, DependencyActionController &Controller) {
+ auto ModCache =
+ makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());
+ CompilerInstance PrescanCI(
+ std::make_shared<CompilerInvocation>(CIPtr->getInvocation()),
+ Worker.PCHContainerOps, std::move(ModCache));
+
+ DiagnosticConsumer DiagConsumer;
+ initializeScanCompilerInstance(PrescanCI, ScanFS, &DiagConsumer,
+ Worker.Service, Worker.DepFS);
+
+ // FIXME: reuse the StableDirs/PrebuiltModuleASTMap computed in initialize().
+ SmallVector<StringRef> PrescanStableDirs = getInitialStableDirs(PrescanCI);
+ if (!computePrebuiltModulesASTMap(PrescanCI, PrescanStableDirs))
+ return false;
+
+ if (PrescanCI.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
+ PrescanCI.getLangOpts().CompilingPCH = true;
+
+ runTUModulePrescan(PrescanCI, Worker.Service, Controller, Compiles);
+ return true;
+}
+
bool CompilerInstanceWithContext::applyAndReport(
ModuleDepCollector &MDC, CompilerInvocation &ModuleInvocation,
DependencyConsumer &Consumer, DependencyActionController &Controller,
@@ -204,6 +232,13 @@ CompilerInstanceWithContext::scanTranslationUnit(
assert(CIPtr && "CIPtr must be initialized before calling this method");
auto &CI = *CIPtr;
+ std::optional<AsyncModuleCompiles> AsyncCompiles;
+ if (Worker.Service.getOpts().AsyncScanModules) {
+ AsyncCompiles.emplace();
+ if (!prescanModulesAsync(*AsyncCompiles, Controller))
+ return nullptr;
+ }
+
auto MDC = initializeScanInstanceDependencyCollector(
CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
Worker.Service, *OriginalInvocation, Controller, PrebuiltModuleASTMap,
diff --git a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
index 68fda9227dfcb..ba85dd5b340df 100644
--- a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
+++ b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
@@ -524,35 +524,6 @@ dependencies::initializeScanInstanceDependencyCollector(
return MDC;
}
-/// Manages (and terminates) the asynchronous compilation of modules.
-class AsyncModuleCompiles {
- std::mutex Mutex;
- bool Stop = false;
- // FIXME: Have the service own a thread pool and use that instead.
- std::vector<std::thread> Compiles;
-
-public:
- /// Registers the module compilation, unless this instance is about to be
- /// destroyed.
- void add(llvm::unique_function<void()> Compile) {
- std::lock_guard<std::mutex> Lock(Mutex);
- if (!Stop)
- Compiles.emplace_back(std::move(Compile));
- }
-
- ~AsyncModuleCompiles() {
- {
- // Prevent registration of further module compiles.
- std::lock_guard<std::mutex> Lock(Mutex);
- Stop = true;
- }
-
- // Wait for outstanding module compiles to finish.
- for (std::thread &Compile : Compiles)
- Compile.join();
- }
-};
-
struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
DependencyScanningService &Service;
DependencyActionController &Controller;
@@ -692,6 +663,14 @@ bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
return true;
}
+void dependencies::runTUModulePrescan(CompilerInstance &PrescanCI,
+ DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles) {
+ SingleTUWithAsyncModuleCompiles Action(Service, Controller, Compiles);
+ (void)PrescanCI.ExecuteAction(Action);
+}
+
bool DependencyScanningAction::runInvocation(
std::string Executable,
std::unique_ptr<CompilerInvocation> OriginalInvocation,
diff --git a/clang/test/ClangScanDeps/modules-async-scan.c b/clang/test/ClangScanDeps/modules-async-scan.c
new file mode 100644
index 0000000000000..656a609567d1d
--- /dev/null
+++ b/clang/test/ClangScanDeps/modules-async-scan.c
@@ -0,0 +1,65 @@
+// Verify that dependency scanning produces identical results with and without
+// -async-scan-modules so we exercise the async scan code path.
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.in > %t/cdb.json
+
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \
+// RUN: -format experimental-full -mode preprocess-dependency-directives \
+// RUN: > %t/sync.json
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 -async-scan-modules \
+// RUN: -format experimental-full -mode preprocess-dependency-directives \
+// RUN: > %t/async.json
+
+// RUN: diff -u %t/sync.json %t/async.json
+
+// Sanity-check that the scan computed the correct module graph.
+// RUN: FileCheck %s < %t/sync.json
+// CHECK-DAG: "name": "A"
+// CHECK-DAG: "name": "B"
+// CHECK-DAG: "name": "C"
+
+//--- cdb.json.in
+
+[{
+ "directory": "DIR",
+ "command": "clang -c DIR/main.c -IDIR -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps",
+ "file": "DIR/main.c"
+}]
+
+//--- module.modulemap
+
+module A {
+ header "a.h"
+}
+
+module B {
+ header "b.h"
+}
+
+module C {
+ header "c.h"
+}
+
+//--- a.h
+
+#include "b.h"
+void a(void);
+
+//--- b.h
+
+void b(void);
+
+//--- c.h
+
+void c(void);
+
+//--- main.c
+
+#include "a.h"
+#include "c.h"
+void m(void) {
+ a();
+ c();
+}
>From 92053d09e6248f8fe8135c76276fc7aa5e39eb86 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Mon, 29 Jun 2026 09:51:49 -0700
Subject: [PATCH 08/13] 4-pre. Splitting CIWC initialization to Invocation init
and CI init.
---
.../CompilerInstanceWithContext.h | 3 +++
.../CompilerInstanceWithContext.cpp | 14 +++++++++++---
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
index 9d93fbf15ee62..4890e06416046 100644
--- a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
+++ b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
@@ -47,6 +47,9 @@ class CompilerInstanceWithContext {
std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS);
+ bool initializeScanInstance(DependencyActionController &Controller,
+ DiagnosticConsumer *DiagConsumer);
+
bool prescanModulesAsync(AsyncModuleCompiles &Compiles,
DependencyActionController &Controller);
diff --git a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
index bd2139329df1a..d890270425a2f 100644
--- a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
+++ b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
@@ -52,6 +52,15 @@ bool CompilerInstanceWithContext::initialize(
return false;
}
+ return initializeScanInstance(
+ Controller, DiagEngineWithDiagOpts->DiagEngine->getClient());
+}
+
+bool CompilerInstanceWithContext::initializeScanInstance(
+ DependencyActionController &Controller, DiagnosticConsumer *DiagConsumer) {
+ assert(OriginalInvocation && ScanFS &&
+ "OriginalInvocation and ScanFS must be set before this call");
+
if (any(Worker.Service.getOpts().OptimizeArgs &
ScanningOptimizations::Macros))
canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
@@ -65,9 +74,8 @@ bool CompilerInstanceWithContext::initialize(
Worker.PCHContainerOps, std::move(ModCache));
auto &CI = *CIPtr;
- initializeScanCompilerInstance(
- CI, ScanFS, DiagEngineWithDiagOpts->DiagEngine->getClient(),
- Worker.Service, Worker.DepFS);
+ initializeScanCompilerInstance(CI, ScanFS, DiagConsumer, Worker.Service,
+ Worker.DepFS);
StableDirs = getInitialStableDirs(CI);
auto MaybePrebuiltModulesASTMap =
>From 8ea9c7520253fafb5d9dff8918f4ab553fd6f918 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Mon, 29 Jun 2026 14:55:29 -0700
Subject: [PATCH 09/13] 4. Remove DependencyScanningAction.
---
.../DependencyScannerImpl.h | 28 -----
.../DependencyScannerImpl.cpp | 117 ------------------
2 files changed, 145 deletions(-)
diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
index 4507400d2da8c..dea2cf5bffc4f 100644
--- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
+++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
@@ -26,37 +26,9 @@ namespace dependencies {
class DependencyScanningService;
class DependencyScanningWorker;
-class DependencyConsumer;
class DependencyActionController;
class DependencyScanningWorkerFilesystem;
-class DependencyScanningAction {
-public:
- DependencyScanningAction(
- DependencyScanningService &Service, StringRef WorkingDirectory,
- DependencyConsumer &Consumer, DependencyActionController &Controller,
- IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS)
- : Service(Service), WorkingDirectory(WorkingDirectory),
- Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)) {}
- bool runInvocation(std::string Executable,
- std::unique_ptr<CompilerInvocation> Invocation,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- DiagnosticConsumer *DiagConsumer);
-
- bool hasScanned() const { return Scanned; }
-
-private:
- DependencyScanningService &Service;
- StringRef WorkingDirectory;
- DependencyConsumer &Consumer;
- DependencyActionController &Controller;
- IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
- std::optional<CompilerInstance> ScanInstanceStorage;
- std::shared_ptr<ModuleDepCollector> MDC;
- bool Scanned = false;
-};
-
// Helper functions and data types.
std::unique_ptr<DiagnosticOptions>
createDiagOptions(ArrayRef<std::string> CommandLine);
diff --git a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
index ba85dd5b340df..58c0deac76fa5 100644
--- a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
+++ b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
@@ -670,120 +670,3 @@ void dependencies::runTUModulePrescan(CompilerInstance &PrescanCI,
SingleTUWithAsyncModuleCompiles Action(Service, Controller, Compiles);
(void)PrescanCI.ExecuteAction(Action);
}
-
-bool DependencyScanningAction::runInvocation(
- std::string Executable,
- std::unique_ptr<CompilerInvocation> OriginalInvocation,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- DiagnosticConsumer *DiagConsumer) {
- // Making sure that we canonicalize the defines early to avoid unnecessary
- // variants in both the scanner and in the resulting explicit command lines.
- if (any(Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
- canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
-
- if (Scanned) {
- CompilerInstance &ScanInstance = *ScanInstanceStorage;
-
- // Scanning runs once for the first -cc1 invocation in a chain of driver
- // jobs. For any dependent jobs, reuse the scanning result and just
- // update the new invocation.
- // FIXME: to support multi-arch builds, each arch requires a separate scan
- if (MDC)
- MDC->applyDiscoveredDependencies(*OriginalInvocation);
-
- bool Success = OriginalInvocation->withCowRef<bool>(
- [&](CowCompilerInvocation &CowOriginalInvocation) {
- return Controller.finalize(ScanInstance, CowOriginalInvocation);
- });
- if (!Success)
- return false;
-
- Consumer.handleBuildCommand(
- {Executable, OriginalInvocation->getCC1CommandLine()});
- return true;
- }
-
- Scanned = true;
-
- // Create a compiler instance to handle the actual work.
- auto ScanInvocation =
- createScanCompilerInvocation(*OriginalInvocation, Service, Controller);
-
- // Quickly discovers and compiles modules for the real scan below.
- std::optional<AsyncModuleCompiles> AsyncCompiles;
- if (Service.getOpts().AsyncScanModules) {
- auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
- auto ScanInstanceStorage = std::make_unique<CompilerInstance>(
- std::make_shared<CompilerInvocation>(*ScanInvocation), PCHContainerOps,
- std::move(ModCache));
- CompilerInstance &ScanInstance = *ScanInstanceStorage;
-
- DiagnosticConsumer DiagConsumer;
- initializeScanCompilerInstance(ScanInstance, FS, &DiagConsumer, Service,
- DepFS);
-
- // FIXME: Do this only once.
- SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
- auto MaybePrebuiltModulesASTMap =
- computePrebuiltModulesASTMap(ScanInstance, StableDirs);
- if (!MaybePrebuiltModulesASTMap)
- return false;
-
- // Normally this would be handled by GeneratePCHAction
- if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
- ScanInstance.getLangOpts().CompilingPCH = true;
-
- AsyncCompiles.emplace();
- SingleTUWithAsyncModuleCompiles Action(Service, Controller, *AsyncCompiles);
- (void)ScanInstance.ExecuteAction(Action);
- }
-
- auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
- ScanInstanceStorage.emplace(std::move(ScanInvocation),
- std::move(PCHContainerOps), std::move(ModCache));
- CompilerInstance &ScanInstance = *ScanInstanceStorage;
-
- initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
- DepFS);
-
- llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
- auto MaybePrebuiltModulesASTMap =
- computePrebuiltModulesASTMap(ScanInstance, StableDirs);
- if (!MaybePrebuiltModulesASTMap)
- return false;
-
- auto DepOutputOpts = createDependencyOutputOptions(*OriginalInvocation);
-
- MDC = initializeScanInstanceDependencyCollector(
- ScanInstance, std::move(DepOutputOpts), Service, *OriginalInvocation,
- Controller, *MaybePrebuiltModulesASTMap, StableDirs);
-
- if (ScanInstance.getDiagnostics().hasErrorOccurred())
- return false;
-
- if (!Controller.initialize(ScanInstance, *OriginalInvocation))
- return false;
-
- ReadPCHAndPreprocessAction Action;
- const bool Result = ScanInstance.ExecuteAction(Action);
-
- if (Result) {
- if (MDC) {
- MDC->run(Consumer);
- MDC->applyDiscoveredDependencies(*OriginalInvocation);
- }
-
- bool Success = OriginalInvocation->withCowRef<bool>(
- [&](CowCompilerInvocation &CowOriginalInvocation) {
- return Controller.finalize(ScanInstance, CowOriginalInvocation);
- });
- if (!Success)
- return false;
-
- Consumer.handleBuildCommand(
- {Executable, OriginalInvocation->getCC1CommandLine()});
- }
-
- return Result;
-}
>From 50559fcbeedcb8bc0b9ca31286ba6ff0dd1bb554 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Tue, 30 Jun 2026 09:07:10 -0700
Subject: [PATCH 10/13] 1. Implement the streaming style by-name scanning API
computeDependenciesByNameWithDrain.
---
.../DependencyScanningWorker.h | 23 +++--
.../clang/Tooling/DependencyScanningTool.h | 23 ++---
.../DependencyScanningWorker.cpp | 33 +++++--
clang/lib/Tooling/DependencyScanningTool.cpp | 95 ++++++++-----------
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 49 +++++-----
5 files changed, 119 insertions(+), 104 deletions(-)
diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
index 428ef43a83044..4d9cab74e7cdb 100644
--- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
@@ -60,6 +60,23 @@ class DependencyScanningWorker {
DiagnosticConsumer &DiagConsumer,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS = nullptr);
+ /// Drain-style by-name scanning over a single cc1 command line. Builds a
+ /// scanning session local to this call, then pulls module names from
+ /// \p getNextInput and delivers each module's dependencies to
+ /// \p deliverResult (std::nullopt on scan failure) until \p getNextInput
+ /// returns std::nullopt. Diagnostics flow to \p DiagConsumer.
+ /// \returns false if session setup failed, true otherwise.
+ bool computeDependenciesByNameWithDrain(
+ StringRef CWD, ArrayRef<std::string> CC1CommandLine,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ DiagnosticConsumer &DiagConsumer,
+ dependencies::DependencyActionController &Controller,
+ const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
+ llvm::function_ref<std::optional<std::string>()> getNextInput,
+ llvm::function_ref<void(StringRef,
+ std::optional<dependencies::TranslationUnitDeps>)>
+ deliverResult);
+
/// Creates the effective VFS that will be used for the scan.
///
/// If provided, OverlayFS will be overlaid on top of the Worker's dependency
@@ -81,12 +98,6 @@ class DependencyScanningWorker {
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
DependencyActionController &Controller);
- void resetCIWC();
-
- bool computeDependenciesByName(StringRef ModuleName,
- DependencyConsumer &Consumer,
- DependencyActionController &Controller);
-
private:
/// The parent dependency scanning service.
DependencyScanningService &Service;
diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h
index 9bda50ee858f9..9c433b5e955bc 100644
--- a/clang/include/clang/Tooling/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanningTool.h
@@ -112,6 +112,16 @@ class DependencyScanningTool {
const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
dependencies::DependencyActionController &Controller);
+ bool computeDependenciesByNameWithDrain(
+ StringRef CWD, ArrayRef<std::string> CommandLine,
+ DiagnosticConsumer &DiagConsumer,
+ dependencies::DependencyActionController &Controller,
+ const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
+ llvm::function_ref<std::optional<std::string>()> getNextInput,
+ llvm::function_ref<void(StringRef,
+ std::optional<dependencies::TranslationUnitDeps>)>
+ deliverResult);
+
/// Returns the worker tracing VFS, if it was requested via the service.
llvm::vfs::TracingFileSystem *getWorkerTracingVFS() const {
return Worker.getTracingVFS();
@@ -119,20 +129,7 @@ class DependencyScanningTool {
dependencies::DependencyScanningWorker &getWorker() { return Worker; }
- llvm::Error initializeForByNameLookup(
- StringRef CWD, ArrayRef<std::string> CommandLine,
- dependencies::DependencyActionController &Controller);
-
- llvm::Expected<dependencies::TranslationUnitDeps>
- computeDependenciesByNameOrError(
- StringRef ModuleName,
- const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
- dependencies::DependencyActionController &Controller);
-
private:
- // Dependency scanning worker has components that depends on the DiagPrinter.
- // Hence the DiagPrinter is declared first. Do not change the ordering.
- std::unique_ptr<dependencies::TextDiagnosticsPrinterWithOutput> DiagPrinter;
dependencies::DependencyScanningWorker Worker;
};
diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
index 999bba284ff39..c2812fe7664b9 100644
--- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
@@ -12,6 +12,7 @@
#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
#include "clang/DependencyScanning/DependencyConsumer.h"
#include "clang/DependencyScanning/DependencyScannerImpl.h"
+#include "clang/DependencyScanning/DependencyScanningUtils.h"
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/Support/VirtualFileSystem.h"
@@ -73,13 +74,33 @@ bool DependencyScanningWorker::initializeCIWC(
return true;
}
-void DependencyScanningWorker::resetCIWC() { CIWC.reset(); }
+bool DependencyScanningWorker::computeDependenciesByNameWithDrain(
+ StringRef CWD, ArrayRef<std::string> CC1CommandLine,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller,
+ const llvm::DenseSet<ModuleID> &AlreadySeen,
+ llvm::function_ref<std::optional<std::string>()> getNextInput,
+ llvm::function_ref<void(StringRef, std::optional<TranslationUnitDeps>)>
+ deliverResult) {
+ auto FS = makeEffectiveVFS(CWD, OverlayFS);
+ auto DiagEngine = std::make_unique<DiagnosticsEngineWithDiagOpts>(
+ CC1CommandLine, FS, DiagConsumer);
+
+ std::optional<CompilerInstanceWithContext> CIWC =
+ CompilerInstanceWithContext::initializeFromCC1Commandline(
+ *this, CWD, CC1CommandLine, std::move(DiagEngine),
+ std::move(OverlayFS), Controller);
+ if (!CIWC)
+ return false;
-bool DependencyScanningWorker::computeDependenciesByName(
- StringRef ModuleName, DependencyConsumer &Consumer,
- DependencyActionController &Controller) {
- assert(CIWC && "initializeCIWC must succeed before calling this method");
- return CIWC->computeDependencies(ModuleName, Consumer, Controller);
+ while (std::optional<std::string> NextInput = getNextInput()) {
+ FullDependencyConsumer Consumer(AlreadySeen);
+ if (CIWC->computeDependencies(*NextInput, Consumer, Controller))
+ deliverResult(*NextInput, Consumer.takeTranslationUnitDeps());
+ else
+ deliverResult(*NextInput, std::nullopt);
+ }
+ return true;
}
bool DependencyScanningWorker::computeDependencies(
diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp
index 2432d20ff24fa..057e9dbfa2340 100644
--- a/clang/lib/Tooling/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanningTool.cpp
@@ -335,9 +335,26 @@ DependencyScanningTool::getModuleDependencies(
StringRef ModuleName, ArrayRef<std::string> CommandLine, StringRef CWD,
const llvm::DenseSet<ModuleID> &AlreadySeen,
DependencyActionController &Controller) {
- if (llvm::Error Err = initializeForByNameLookup(CWD, CommandLine, Controller))
- return std::move(Err);
- return computeDependenciesByNameOrError(ModuleName, AlreadySeen, Controller);
+ TextDiagnosticsPrinterWithOutput DiagPrinter(CommandLine);
+
+ std::optional<TranslationUnitDeps> Result;
+ bool Pulled = false;
+ auto getNextName = [&]() -> std::optional<std::string> {
+ if (Pulled)
+ return std::nullopt;
+ Pulled = true;
+ return ModuleName.str();
+ };
+ auto deliverResult = [&](StringRef, std::optional<TranslationUnitDeps> R) {
+ Result = std::move(R);
+ };
+
+ if (!computeDependenciesByNameWithDrain(
+ CWD, CommandLine, DiagPrinter.DiagPrinter, Controller, AlreadySeen,
+ getNextName, deliverResult) ||
+ !Result)
+ return makeErrorFromDiagnosticsOS(DiagPrinter);
+ return std::move(*Result);
}
static std::optional<SmallVector<std::string, 0>>
@@ -363,59 +380,31 @@ getFirstCC1CommandLine(ArrayRef<std::string> CommandLine,
return std::nullopt;
}
-static bool
-initializeWorkerForByNameLookup(DependencyScanningTool &Tool, StringRef CWD,
- ArrayRef<std::string> CommandLine,
- DependencyActionController &Controller,
- DiagnosticConsumer &DC) {
+bool DependencyScanningTool::computeDependenciesByNameWithDrain(
+ StringRef CWD, ArrayRef<std::string> CommandLine,
+ DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller,
+ const llvm::DenseSet<ModuleID> &AlreadySeen,
+ llvm::function_ref<std::optional<std::string>()> getNextInput,
+ llvm::function_ref<void(StringRef, std::optional<TranslationUnitDeps>)>
+ deliverResult) {
auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(CommandLine);
- auto FS = Tool.getWorker().makeEffectiveVFS(CWD, OverlayFS);
-
- auto DiagEngineWithCmdAndOpts =
- std::make_unique<DiagnosticsEngineWithDiagOpts>(ModifiedCommandLine, FS,
- DC);
+ std::vector<std::string> CC1CommandLine;
if (ModifiedCommandLine.size() >= 2 && ModifiedCommandLine[1] == "-cc1") {
- // The input command line is already a -cc1 invocation; initialize the
- // compiler instance directly from it.
- return Tool.getWorker().initializeCIWC(CWD, ModifiedCommandLine,
- std::move(DiagEngineWithCmdAndOpts),
- std::move(OverlayFS), Controller);
+ CC1CommandLine = std::move(ModifiedCommandLine);
+ } else {
+ // Driver-style (or ill-formed): lower to a cc1 command line, or diagnose.
+ auto FS = Worker.makeEffectiveVFS(CWD, OverlayFS);
+ DiagnosticsEngineWithDiagOpts DiagEngine(ModifiedCommandLine, FS,
+ DiagConsumer);
+ auto MaybeFirstCC1 =
+ getFirstCC1CommandLine(ModifiedCommandLine, *DiagEngine.DiagEngine, FS);
+ if (!MaybeFirstCC1)
+ return false;
+ CC1CommandLine.assign(MaybeFirstCC1->begin(), MaybeFirstCC1->end());
}
- // The input command line is either a driver-style command line, or
- // ill-formed. In this case, we will first call the Driver to build a -cc1
- // command line for this compilation or diagnose any ill-formed input.
- const auto MaybeFirstCC1 = getFirstCC1CommandLine(
- ModifiedCommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, FS);
- if (!MaybeFirstCC1)
- return false;
-
- std::vector<std::string> CC1CommandLine(MaybeFirstCC1->begin(),
- MaybeFirstCC1->end());
- return Tool.getWorker().initializeCIWC(CWD, CC1CommandLine,
- std::move(DiagEngineWithCmdAndOpts),
- std::move(OverlayFS), Controller);
-}
-
-llvm::Error DependencyScanningTool::initializeForByNameLookup(
- StringRef CWD, ArrayRef<std::string> CommandLine,
- DependencyActionController &Controller) {
- Worker.resetCIWC();
- DiagPrinter = std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine);
- if (!initializeWorkerForByNameLookup(*this, CWD, CommandLine, Controller,
- DiagPrinter->DiagPrinter))
- return makeErrorFromDiagnosticsOS(*DiagPrinter);
- return llvm::Error::success();
-}
-
-llvm::Expected<TranslationUnitDeps>
-DependencyScanningTool::computeDependenciesByNameOrError(
- StringRef ModuleName, const llvm::DenseSet<ModuleID> &AlreadySeen,
- DependencyActionController &Controller) {
- FullDependencyConsumer Consumer(AlreadySeen);
- DiagPrinter->DiagnosticOutput.clear();
- if (Worker.computeDependenciesByName(ModuleName, Consumer, Controller))
- return Consumer.takeTranslationUnitDeps();
- return makeErrorFromDiagnosticsOS(*DiagPrinter);
+ return Worker.computeDependenciesByNameWithDrain(
+ CWD, CC1CommandLine, std::move(OverlayFS), DiagConsumer, Controller,
+ AlreadySeen, getNextInput, deliverResult);
}
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index b586b1fd83111..278ea3adaf94b 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -659,16 +659,6 @@ static bool handleModuleResult(StringRef ModuleName,
return false;
}
-static void handleErrorWithInfoString(StringRef Info, llvm::Error E,
- SharedStream &OS, SharedStream &Errs) {
- llvm::handleAllErrors(std::move(E), [&Info, &Errs](llvm::StringError &Err) {
- Errs.applyLocked([&](raw_ostream &OS) {
- OS << "Error: " << Info << ":\n";
- OS << Err.getMessage();
- });
- });
-}
-
class P1689Deps {
public:
void printDependencies(raw_ostream &OS) {
@@ -1115,23 +1105,30 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
LocalIndex, DependencyOS, Errs))
HadErrors = true;
} else {
- if (llvm::Error Err = WorkerTool.initializeForByNameLookup(
- CWD, Input->CommandLine, Controller)) {
- handleErrorWithInfoString(
- "Compiler instance with context setup error", std::move(Err),
- DependencyOS, Errs);
- HadErrors = true;
- continue;
- }
-
- for (auto N : Names) {
- auto MaybeModuleDepsGraph =
- WorkerTool.computeDependenciesByNameOrError(
- N, AlreadySeenModules, Controller);
- if (handleModuleResult(N, MaybeModuleDepsGraph, *FD, LocalIndex,
- DependencyOS, Errs)) {
+ unsigned NameIdx = 0;
+ auto getNextName = [&]() -> std::optional<std::string> {
+ if (NameIdx < Names.size())
+ return Names[NameIdx++].str();
+ return std::nullopt;
+ };
+ auto deliverResult = [&](StringRef Name,
+ std::optional<TranslationUnitDeps> Result) {
+ llvm::Expected<TranslationUnitDeps> MaybeTUDeps =
+ Result ? llvm::Expected<TranslationUnitDeps>(std::move(*Result))
+ : llvm::Expected<TranslationUnitDeps>(
+ llvm::make_error<llvm::StringError>(
+ S, llvm::inconvertibleErrorCode()));
+ if (handleModuleResult(Name, MaybeTUDeps, *FD, LocalIndex,
+ DependencyOS, Errs))
HadErrors = true;
- }
+ S.clear();
+ };
+
+ if (!WorkerTool.computeDependenciesByNameWithDrain(
+ CWD, Input->CommandLine, DiagConsumer, Controller,
+ AlreadySeenModules, getNextName, deliverResult)) {
+ handleDiagnostics(Filename, S, Errs);
+ HadErrors = true;
}
}
} else {
>From 254e458333c320ad0bd0f390a3d33f1e52cb5694 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Tue, 30 Jun 2026 10:00:12 -0700
Subject: [PATCH 11/13] 2. Remove the CIWC member from DependencyScanningWorker
for TU scanning. CIWC is now method scoped only.
---
.../DependencyScanningWorker.h | 8 ------
.../DependencyScanningWorker.cpp | 27 +++++--------------
2 files changed, 7 insertions(+), 28 deletions(-)
diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
index 4d9cab74e7cdb..7487f7a4b7c2a 100644
--- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
@@ -92,12 +92,6 @@ class DependencyScanningWorker {
return TracingFS.get();
}
- bool initializeCIWC(
- StringRef CWD, ArrayRef<std::string> CC1CommandLine,
- std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
- DependencyActionController &Controller);
-
private:
/// The parent dependency scanning service.
DependencyScanningService &Service;
@@ -108,8 +102,6 @@ class DependencyScanningWorker {
/// The tracing VFS overlaid on top of the base VFS.
IntrusiveRefCntPtr<llvm::vfs::TracingFileSystem> TracingFS;
- std::unique_ptr<CompilerInstanceWithContext> CIWC;
-
friend class CompilerInstanceWithContext;
};
} // end namespace dependencies
diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
index c2812fe7664b9..9d1e37388ea6b 100644
--- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
@@ -59,21 +59,6 @@ DependencyScanningWorker::makeEffectiveVFS(
return FS;
}
-bool DependencyScanningWorker::initializeCIWC(
- StringRef CWD, ArrayRef<std::string> CC1CommandLine,
- std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
- DependencyActionController &Controller) {
- CIWC.reset();
- auto Result = CompilerInstanceWithContext::initializeFromCC1Commandline(
- *this, CWD, CC1CommandLine, std::move(DiagEngineWithDiagOpts),
- std::move(OverlayFS), Controller);
- if (!Result)
- return false;
- CIWC = std::make_unique<CompilerInstanceWithContext>(std::move(*Result));
- return true;
-}
-
bool DependencyScanningWorker::computeDependenciesByNameWithDrain(
StringRef CWD, ArrayRef<std::string> CC1CommandLine,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
@@ -113,6 +98,7 @@ bool DependencyScanningWorker::computeDependencies(
bool Scanned = false;
std::shared_ptr<ModuleDepCollector> MDC;
+ std::optional<CompilerInstanceWithContext> CIWC;
const bool Success = llvm::all_of(CommandLines, [&](const auto &Cmd) {
if (StringRef(Cmd[1]) != "-cc1") {
// Non-clang command. Just pass through to the dependency consumer.
@@ -125,10 +111,12 @@ bool DependencyScanningWorker::computeDependencies(
std::make_unique<DiagnosticsEngineWithDiagOpts>(Cmd, FS, DiagConsumer);
if (!Scanned) {
Scanned = true;
- if (!initializeCIWC(WorkingDirectory, Cmd,
- std::move(DiagEngineWithDiagOpts), OverlayFS,
- Controller))
+ auto Result = CompilerInstanceWithContext::initializeFromCC1Commandline(
+ *this, WorkingDirectory, Cmd, std::move(DiagEngineWithDiagOpts),
+ OverlayFS, Controller);
+ if (!Result)
return false;
+ CIWC.emplace(std::move(*Result));
MDC = CIWC->scanTranslationUnit(DepConsumer, Controller);
return MDC != nullptr;
}
@@ -139,8 +127,7 @@ bool DependencyScanningWorker::computeDependencies(
if (!Invocation)
return false;
- if (!Invocation)
- return false;
+ assert(CIWC && "Must have an initialized CIWC");
return CIWC->applyAndReport(*MDC, *Invocation, DepConsumer, Controller,
Cmd.front());
});
>From 96543decabb580c9e3ec0aa73b05657ed0ed3f49 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Mon, 6 Jul 2026 12:48:59 -0700
Subject: [PATCH 12/13] 1. Fold CompilerInstanceWithContext into
DependencyScanningWorker.cpp
---
.../CompilerInstanceWithContext.h | 100 ------
.../DependencyScanningWorker.h | 10 +
.../clang/Tooling/DependencyScanningTool.h | 3 -
clang/lib/DependencyScanning/CMakeLists.txt | 1 -
.../CompilerInstanceWithContext.cpp | 270 ----------------
.../DependencyScanningWorker.cpp | 293 +++++++++++++++++-
clang/lib/Tooling/DependencyScanningTool.cpp | 3 +-
7 files changed, 303 insertions(+), 377 deletions(-)
delete mode 100644 clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
delete mode 100644 clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
diff --git a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h b/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
deleted file mode 100644
index 4890e06416046..0000000000000
--- a/clang/include/clang/DependencyScanning/CompilerInstanceWithContext.h
+++ /dev/null
@@ -1,100 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_DEPENDENCYSCANNING_COMPILERINSTANCEWITHCONTEXT_H
-#define LLVM_CLANG_DEPENDENCYSCANNING_COMPILERINSTANCEWITHCONTEXT_H
-
-#include "clang/DependencyScanning/DependencyScanningWorker.h"
-
-namespace clang {
-namespace dependencies {
-class CompilerInstanceWithContext {
- // Context
- DependencyScanningWorker &Worker;
- llvm::StringRef CWD;
- std::vector<std::string> CommandLine;
-
- // Context - compiler invocation
- std::unique_ptr<CompilerInvocation> OriginalInvocation;
-
- // Context - output options
- std::unique_ptr<DependencyOutputOptions> OutputOpts;
-
- // Context - stable directory handling
- llvm::SmallVector<StringRef> StableDirs;
- PrebuiltModulesAttrsMap PrebuiltModuleASTMap;
-
- // Context - used by AsyncScan's prescan pass
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS;
-
- // Compiler Instance
- std::unique_ptr<CompilerInstance> CIPtr;
-
- // Source location offset.
- int32_t SrcLocOffset = 0;
-
- CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD,
- ArrayRef<std::string> CMD)
- : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {}
-
- bool initialize(
- DependencyActionController &Controller,
- std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS);
-
- bool initializeScanInstance(DependencyActionController &Controller,
- DiagnosticConsumer *DiagConsumer);
-
- bool prescanModulesAsync(AsyncModuleCompiles &Compiles,
- DependencyActionController &Controller);
-
-public:
- /// @brief Initialize the tool's compiler instance from the cc1 commandline.
- /// @param Worker The dependency scanning worker to initialize the compiler
- /// instance.
- /// @param CWD The current working directory.
- /// @param CC1CommandLine A cc1 command.
- /// @param DiagEngineWithDiagOpts The diagnostic engine used during scan.
- /// @param OverlayFS An overlay FS containing the input file, which may be
- /// from an in-memory buffer.
- /// @param Controller A dependency action controller to gather some results.
- static std::optional<CompilerInstanceWithContext>
- initializeFromCC1Commandline(
- DependencyScanningWorker &Worker, StringRef CWD,
- ArrayRef<std::string> CC1CommandLine,
- std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
- DependencyActionController &Controller);
-
- bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer,
- DependencyActionController &Controller);
-
- // MaxNumOfQueries is the upper limit of the number of names the by-name
- // scanning API (computeDependencies) can support after a
- // CompilerInstanceWithContext is initialized. At the time of this commit, the
- // estimated number of total unique importable names is around 3000 from
- // Apple's SDKs. We usually import them in parallel, so it is unlikely that
- // all names are all scanned by the same dependency scanning worker. Therefore
- // the 64k (20x bigger than our estimate) size is sufficient to hold the
- // unique source locations to report diagnostics per worker.
- static const int32_t MaxNumOfQueries = 1 << 16;
-
- std::shared_ptr<ModuleDepCollector>
- scanTranslationUnit(DependencyConsumer &Consumer,
- DependencyActionController &Controller);
-
- bool applyAndReport(ModuleDepCollector &MDC,
- CompilerInvocation &ModuleInvocation,
- DependencyConsumer &Consumer,
- DependencyActionController &Controller,
- StringRef Executable);
-};
-} // namespace dependencies
-} // namespace clang
-
-#endif
\ No newline at end of file
diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
index 7487f7a4b7c2a..ecf6d97f7e847 100644
--- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
@@ -92,6 +92,16 @@ class DependencyScanningWorker {
return TracingFS.get();
}
+ // MaxNumOfByNameQueries is the upper limit of the number of names the by-name
+ // scanning API (computeDependenciesByNameWithDrain) can drain per call. At
+ // the time of this commit, the estimated number of total unique importable
+ // names is around 3000 from Apple's SDKs. We usually import them in parallel,
+ // so it is unlikely that all names are all scanned by the same dependency
+ // scanning worker. Therefore the 64k (20x bigger than our estimate) size is
+ // sufficient to hold the unique source locations to report diagnostics per
+ // worker.
+ static const int32_t MaxNumOfByNameQueries = 1 << 16;
+
private:
/// The parent dependency scanning service.
DependencyScanningService &Service;
diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h
index 9c433b5e955bc..c52b0330acae2 100644
--- a/clang/include/clang/Tooling/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanningTool.h
@@ -104,9 +104,6 @@ class DependencyScanningTool {
/// Given a compilation context specified via the Clang driver command-line,
/// gather modular dependencies of module with the given name, and return the
/// information needed for explicit build.
- /// TODO: this method should be removed as soon as Swift and our C-APIs adopt
- /// CompilerInstanceWithContext. We are keeping it here so that it is easier
- /// to coordinate with Swift and C-API changes.
llvm::Expected<dependencies::TranslationUnitDeps> getModuleDependencies(
StringRef ModuleName, ArrayRef<std::string> CommandLine, StringRef CWD,
const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
diff --git a/clang/lib/DependencyScanning/CMakeLists.txt b/clang/lib/DependencyScanning/CMakeLists.txt
index b7d85e6a09c02..015c6bcb12326 100644
--- a/clang/lib/DependencyScanning/CMakeLists.txt
+++ b/clang/lib/DependencyScanning/CMakeLists.txt
@@ -6,7 +6,6 @@ set(LLVM_LINK_COMPONENTS
)
add_clang_library(clangDependencyScanning
- CompilerInstanceWithContext.cpp
DependencyGraph.cpp
DependencyScanningFilesystem.cpp
DependencyScanningService.cpp
diff --git a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp b/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
deleted file mode 100644
index d890270425a2f..0000000000000
--- a/clang/lib/DependencyScanning/CompilerInstanceWithContext.cpp
+++ /dev/null
@@ -1,270 +0,0 @@
-//===- CompilerInstanceWithContext.cpp - CI for dependency scanning -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/DiagnosticFrontend.h"
-#include "clang/DependencyScanning/DependencyActionController.h"
-#include "clang/DependencyScanning/DependencyConsumer.h"
-#include "clang/DependencyScanning/DependencyScannerImpl.h"
-#include "clang/DependencyScanning/InProcessModuleCache.h"
-#include "clang/Frontend/CompilerInvocation.h"
-#include "clang/Frontend/FrontendActions.h"
-#include "clang/Frontend/FrontendOptions.h"
-#include "llvm/ADT/ScopeExit.h"
-#include <optional>
-
-using namespace clang;
-using namespace dependencies;
-
-std::optional<CompilerInstanceWithContext>
-CompilerInstanceWithContext::initializeFromCC1Commandline(
- DependencyScanningWorker &Worker, StringRef CWD,
- ArrayRef<std::string> CC1CommandLine,
- std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
- DependencyActionController &Controller) {
- CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine);
- if (!CIWC.initialize(Controller, std::move(DiagEngineWithDiagOpts),
- std::move(OverlayFS)))
- return std::nullopt;
- return std::move(CIWC);
-}
-
-bool CompilerInstanceWithContext::initialize(
- DependencyActionController &Controller,
- std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
- assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
- ScanFS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
-
- OriginalInvocation = createCompilerInvocation(
- CommandLine, *DiagEngineWithDiagOpts->DiagEngine);
- if (!OriginalInvocation) {
- DiagEngineWithDiagOpts->DiagEngine->Report(
- diag::err_fe_expected_compiler_job)
- << llvm::join(CommandLine, " ");
- return false;
- }
-
- return initializeScanInstance(
- Controller, DiagEngineWithDiagOpts->DiagEngine->getClient());
-}
-
-bool CompilerInstanceWithContext::initializeScanInstance(
- DependencyActionController &Controller, DiagnosticConsumer *DiagConsumer) {
- assert(OriginalInvocation && ScanFS &&
- "OriginalInvocation and ScanFS must be set before this call");
-
- if (any(Worker.Service.getOpts().OptimizeArgs &
- ScanningOptimizations::Macros))
- canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
-
- // Create the CompilerInstance.
- std::shared_ptr<ModuleCache> ModCache =
- makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());
- CIPtr = std::make_unique<CompilerInstance>(
- createScanCompilerInvocation(*OriginalInvocation, Worker.Service,
- Controller),
- Worker.PCHContainerOps, std::move(ModCache));
- auto &CI = *CIPtr;
-
- initializeScanCompilerInstance(CI, ScanFS, DiagConsumer, Worker.Service,
- Worker.DepFS);
-
- StableDirs = getInitialStableDirs(CI);
- auto MaybePrebuiltModulesASTMap =
- computePrebuiltModulesASTMap(CI, StableDirs);
- if (!MaybePrebuiltModulesASTMap)
- return false;
-
- PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
- OutputOpts = createDependencyOutputOptions(*OriginalInvocation);
-
- // We do not create the target in initializeScanCompilerInstance because
- // setting it here is unique for by-name lookups. We create the target only
- // once here, and the information is reused for all computeDependencies calls.
- // We do not need to call createTarget explicitly if we go through
- // CompilerInstance::ExecuteAction to perform scanning.
- CI.createTarget();
-
- return true;
-}
-
-bool CompilerInstanceWithContext::prescanModulesAsync(
- AsyncModuleCompiles &Compiles, DependencyActionController &Controller) {
- auto ModCache =
- makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());
- CompilerInstance PrescanCI(
- std::make_shared<CompilerInvocation>(CIPtr->getInvocation()),
- Worker.PCHContainerOps, std::move(ModCache));
-
- DiagnosticConsumer DiagConsumer;
- initializeScanCompilerInstance(PrescanCI, ScanFS, &DiagConsumer,
- Worker.Service, Worker.DepFS);
-
- // FIXME: reuse the StableDirs/PrebuiltModuleASTMap computed in initialize().
- SmallVector<StringRef> PrescanStableDirs = getInitialStableDirs(PrescanCI);
- if (!computePrebuiltModulesASTMap(PrescanCI, PrescanStableDirs))
- return false;
-
- if (PrescanCI.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
- PrescanCI.getLangOpts().CompilingPCH = true;
-
- runTUModulePrescan(PrescanCI, Worker.Service, Controller, Compiles);
- return true;
-}
-
-bool CompilerInstanceWithContext::applyAndReport(
- ModuleDepCollector &MDC, CompilerInvocation &ModuleInvocation,
- DependencyConsumer &Consumer, DependencyActionController &Controller,
- StringRef Executable) {
- MDC.applyDiscoveredDependencies(ModuleInvocation);
-
- bool Success = ModuleInvocation.withCowRef<bool>(
- [&](CowCompilerInvocation &CowModuleInvocation) {
- return Controller.finalize(*CIPtr, CowModuleInvocation);
- });
- if (!Success)
- return false;
-
- Consumer.handleBuildCommand(
- {Executable.str(), ModuleInvocation.getCC1CommandLine()});
- return true;
-}
-
-bool CompilerInstanceWithContext::computeDependencies(
- StringRef ModuleName, DependencyConsumer &Consumer,
- DependencyActionController &Controller) {
- if (SrcLocOffset >= MaxNumOfQueries)
- llvm::report_fatal_error("exceeded maximum by-name scans for worker");
-
- assert(CIPtr && "CIPtr must be initialized before calling this method");
- auto &CI = *CIPtr;
-
- // We need to reset the diagnostics, so that the diagnostics issued
- // during a previous computeDependencies call do not affect the current call.
- // If we do not reset, we may inherit fatal errors from a previous call.
- CI.getDiagnostics().Reset();
-
- // We create this cleanup object because computeDependencies may exit
- // early with errors.
- llvm::scope_exit CleanUp([&]() {
- CI.clearDependencyCollectors();
- // The preprocessor may not be created at the entry of this method,
- // but it must have been created when this method returns, whether
- // there are errors during scanning or not.
- CI.getPreprocessor().removePPCallbacks();
- });
-
- auto MDC = initializeScanInstanceDependencyCollector(
- CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
- Worker.Service,
- /* The MDC's constructor makes a copy of the OriginalInvocation, so
- we can pass it in without worrying that it might be changed across
- invocations of computeDependencies. */
- *OriginalInvocation, Controller, PrebuiltModuleASTMap, StableDirs);
-
- CompilerInvocation ModuleInvocation(*OriginalInvocation);
- if (!Controller.initialize(CI, ModuleInvocation))
- return false;
-
- if (!SrcLocOffset) {
- // When SrcLocOffset is zero, we are at the beginning of the fake source
- // file. In this case, we call BeginSourceFile to initialize.
- std::unique_ptr<FrontendAction> Action =
- std::make_unique<PreprocessOnlyAction>();
- auto *InputFile = CI.getFrontendOpts().Inputs.begin();
- bool ActionBeginSucceeded = Action->BeginSourceFile(CI, *InputFile);
- assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");
- (void)ActionBeginSucceeded;
- }
-
- Preprocessor &PP = CI.getPreprocessor();
- SourceManager &SM = PP.getSourceManager();
- FileID MainFileID = SM.getMainFileID();
- SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
- SourceLocation IDLocation = FileStart.getLocWithOffset(SrcLocOffset);
- PPCallbacks *CB = nullptr;
- if (!SrcLocOffset) {
- // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
- // the preprocessor.
- bool PPFailed = PP.EnterSourceFile(MainFileID, nullptr, SourceLocation());
- assert(!PPFailed && "Preprocess must be able to enter the main file.");
- (void)PPFailed;
- CB = MDC->getPPCallbacks();
- } else {
- // When SrcLocOffset is non-zero, the preprocessor has already been
- // initialized through a previous call of computeDependencies. We want to
- // preserve the PP's state, hence we do not call EnterSourceFile again.
- MDC->attachToPreprocessor(PP);
- CB = MDC->getPPCallbacks();
-
- FileID PrevFID;
- SrcMgr::CharacteristicKind FileType = SM.getFileCharacteristic(IDLocation);
- CB->LexedFileChanged(MainFileID,
- PPChainedCallbacks::LexedFileChangeReason::EnterFile,
- FileType, PrevFID, IDLocation);
- }
-
- // FIXME: Scan modules asynchronously here as well.
-
- SrcLocOffset++;
- SmallVector<IdentifierLoc, 2> Path;
- IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
- Path.emplace_back(IDLocation, ModuleID);
- auto ModResult = CI.loadModule(IDLocation, Path, Module::Hidden, false);
-
- assert(CB && "Must have PPCallbacks after module loading");
- CB->moduleImport(SourceLocation(), Path, ModResult);
-
- if (!ModResult)
- return false;
-
- if (CI.getDiagnostics().hasErrorOccurred())
- return false;
-
- MDC->run(Consumer);
- return applyAndReport(*MDC, ModuleInvocation, Consumer, Controller,
- CommandLine[0]);
-}
-
-std::shared_ptr<ModuleDepCollector>
-CompilerInstanceWithContext::scanTranslationUnit(
- DependencyConsumer &Consumer, DependencyActionController &Controller) {
- assert(CIPtr && "CIPtr must be initialized before calling this method");
- auto &CI = *CIPtr;
-
- std::optional<AsyncModuleCompiles> AsyncCompiles;
- if (Worker.Service.getOpts().AsyncScanModules) {
- AsyncCompiles.emplace();
- if (!prescanModulesAsync(*AsyncCompiles, Controller))
- return nullptr;
- }
-
- auto MDC = initializeScanInstanceDependencyCollector(
- CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
- Worker.Service, *OriginalInvocation, Controller, PrebuiltModuleASTMap,
- StableDirs);
-
- if (CI.getDiagnostics().hasErrorOccurred())
- return nullptr;
-
- if (!Controller.initialize(CI, *OriginalInvocation))
- return nullptr;
-
- ReadPCHAndPreprocessAction Action;
- if (!CI.ExecuteAction(Action))
- return nullptr;
-
- MDC->run(Consumer);
- if (!applyAndReport(*MDC, *OriginalInvocation, Consumer, Controller,
- CommandLine[0]))
- return nullptr;
- return MDC;
-}
diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
index 9d1e37388ea6b..853caf48262e4 100644
--- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
@@ -9,17 +9,308 @@
#include "clang/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticFrontend.h"
-#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
+#include "clang/DependencyScanning/DependencyActionController.h"
#include "clang/DependencyScanning/DependencyConsumer.h"
#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/DependencyScanning/DependencyScanningUtils.h"
+#include "clang/DependencyScanning/InProcessModuleCache.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/FrontendOptions.h"
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/ScopeExit.h"
#include "llvm/Support/VirtualFileSystem.h"
+#include <optional>
using namespace clang;
using namespace dependencies;
+namespace clang {
+namespace dependencies {
+class CompilerInstanceWithContext {
+ // Context
+ DependencyScanningWorker &Worker;
+ llvm::StringRef CWD;
+ std::vector<std::string> CommandLine;
+
+ // Context - compiler invocation
+ std::unique_ptr<CompilerInvocation> OriginalInvocation;
+
+ // Context - output options
+ std::unique_ptr<DependencyOutputOptions> OutputOpts;
+
+ // Context - stable directory handling
+ llvm::SmallVector<StringRef> StableDirs;
+ PrebuiltModulesAttrsMap PrebuiltModuleASTMap;
+
+ // Context - used by AsyncScan's prescan pass
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS;
+
+ // Compiler Instance
+ std::unique_ptr<CompilerInstance> CIPtr;
+
+ // Source location offset.
+ int32_t SrcLocOffset = 0;
+
+ CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD,
+ ArrayRef<std::string> CMD)
+ : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {}
+
+ bool initialize(
+ DependencyActionController &Controller,
+ std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
+ assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
+ ScanFS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
+
+ OriginalInvocation = createCompilerInvocation(
+ CommandLine, *DiagEngineWithDiagOpts->DiagEngine);
+ if (!OriginalInvocation) {
+ DiagEngineWithDiagOpts->DiagEngine->Report(
+ diag::err_fe_expected_compiler_job)
+ << llvm::join(CommandLine, " ");
+ return false;
+ }
+
+ return initializeScanInstance(
+ Controller, DiagEngineWithDiagOpts->DiagEngine->getClient());
+ }
+
+ bool initializeScanInstance(DependencyActionController &Controller,
+ DiagnosticConsumer *DiagConsumer) {
+ assert(OriginalInvocation && ScanFS &&
+ "OriginalInvocation and ScanFS must be set before this call");
+
+ if (any(Worker.Service.getOpts().OptimizeArgs &
+ ScanningOptimizations::Macros))
+ canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
+
+ // Create the CompilerInstance.
+ std::shared_ptr<ModuleCache> ModCache =
+ makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());
+ CIPtr = std::make_unique<CompilerInstance>(
+ createScanCompilerInvocation(*OriginalInvocation, Worker.Service,
+ Controller),
+ Worker.PCHContainerOps, std::move(ModCache));
+ auto &CI = *CIPtr;
+
+ initializeScanCompilerInstance(CI, ScanFS, DiagConsumer, Worker.Service,
+ Worker.DepFS);
+
+ StableDirs = getInitialStableDirs(CI);
+ auto MaybePrebuiltModulesASTMap =
+ computePrebuiltModulesASTMap(CI, StableDirs);
+ if (!MaybePrebuiltModulesASTMap)
+ return false;
+
+ PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
+ OutputOpts = createDependencyOutputOptions(*OriginalInvocation);
+
+ // We do not create the target in initializeScanCompilerInstance because
+ // setting it here is unique for by-name lookups. We create the target only
+ // once here, and the information is reused for all computeDependencies
+ // calls. We do not need to call createTarget explicitly if we go through
+ // CompilerInstance::ExecuteAction to perform scanning.
+ CI.createTarget();
+
+ return true;
+ }
+
+ bool prescanModulesAsync(AsyncModuleCompiles &Compiles,
+ DependencyActionController &Controller) {
+ auto ModCache =
+ makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());
+ CompilerInstance PrescanCI(
+ std::make_shared<CompilerInvocation>(CIPtr->getInvocation()),
+ Worker.PCHContainerOps, std::move(ModCache));
+
+ DiagnosticConsumer DiagConsumer;
+ initializeScanCompilerInstance(PrescanCI, ScanFS, &DiagConsumer,
+ Worker.Service, Worker.DepFS);
+
+ // FIXME: reuse the StableDirs/PrebuiltModuleASTMap computed in
+ // initialize().
+ SmallVector<StringRef> PrescanStableDirs = getInitialStableDirs(PrescanCI);
+ if (!computePrebuiltModulesASTMap(PrescanCI, PrescanStableDirs))
+ return false;
+
+ if (PrescanCI.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
+ PrescanCI.getLangOpts().CompilingPCH = true;
+
+ runTUModulePrescan(PrescanCI, Worker.Service, Controller, Compiles);
+ return true;
+ }
+
+public:
+ static std::optional<CompilerInstanceWithContext>
+ initializeFromCC1Commandline(
+ DependencyScanningWorker &Worker, StringRef CWD,
+ ArrayRef<std::string> CC1CommandLine,
+ std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
+ DependencyActionController &Controller) {
+ CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine);
+ if (!CIWC.initialize(Controller, std::move(DiagEngineWithDiagOpts),
+ std::move(OverlayFS)))
+ return std::nullopt;
+ return std::move(CIWC);
+ }
+
+ bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer,
+ DependencyActionController &Controller) {
+ if (SrcLocOffset >= DependencyScanningWorker::MaxNumOfByNameQueries)
+ llvm::report_fatal_error("exceeded maximum by-name scans for worker");
+
+ assert(CIPtr && "CIPtr must be initialized before calling this method");
+ auto &CI = *CIPtr;
+
+ // We need to reset the diagnostics, so that the diagnostics issued
+ // during a previous computeDependencies call do not affect the current
+ // call. If we do not reset, we may inherit fatal errors from a previous
+ // call.
+ CI.getDiagnostics().Reset();
+
+ // We create this cleanup object because computeDependencies may exit
+ // early with errors.
+ llvm::scope_exit CleanUp([&]() {
+ CI.clearDependencyCollectors();
+ // The preprocessor may not be created at the entry of this method,
+ // but it must have been created when this method returns, whether
+ // there are errors during scanning or not.
+ CI.getPreprocessor().removePPCallbacks();
+ });
+
+ auto MDC = initializeScanInstanceDependencyCollector(
+ CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
+ Worker.Service,
+ /* The MDC's constructor makes a copy of the OriginalInvocation, so
+ we can pass it in without worrying that it might be changed across
+ invocations of computeDependencies. */
+ *OriginalInvocation, Controller, PrebuiltModuleASTMap, StableDirs);
+
+ CompilerInvocation ModuleInvocation(*OriginalInvocation);
+ if (!Controller.initialize(CI, ModuleInvocation))
+ return false;
+
+ if (!SrcLocOffset) {
+ // When SrcLocOffset is zero, we are at the beginning of the fake source
+ // file. In this case, we call BeginSourceFile to initialize.
+ std::unique_ptr<FrontendAction> Action =
+ std::make_unique<PreprocessOnlyAction>();
+ auto *InputFile = CI.getFrontendOpts().Inputs.begin();
+ bool ActionBeginSucceeded = Action->BeginSourceFile(CI, *InputFile);
+ assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");
+ (void)ActionBeginSucceeded;
+ }
+
+ Preprocessor &PP = CI.getPreprocessor();
+ SourceManager &SM = PP.getSourceManager();
+ FileID MainFileID = SM.getMainFileID();
+ SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
+ SourceLocation IDLocation = FileStart.getLocWithOffset(SrcLocOffset);
+ PPCallbacks *CB = nullptr;
+ if (!SrcLocOffset) {
+ // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
+ // the preprocessor.
+ bool PPFailed = PP.EnterSourceFile(MainFileID, nullptr, SourceLocation());
+ assert(!PPFailed && "Preprocess must be able to enter the main file.");
+ (void)PPFailed;
+ CB = MDC->getPPCallbacks();
+ } else {
+ // When SrcLocOffset is non-zero, the preprocessor has already been
+ // initialized through a previous call of computeDependencies. We want to
+ // preserve the PP's state, hence we do not call EnterSourceFile again.
+ MDC->attachToPreprocessor(PP);
+ CB = MDC->getPPCallbacks();
+
+ FileID PrevFID;
+ SrcMgr::CharacteristicKind FileType =
+ SM.getFileCharacteristic(IDLocation);
+ CB->LexedFileChanged(MainFileID,
+ PPChainedCallbacks::LexedFileChangeReason::EnterFile,
+ FileType, PrevFID, IDLocation);
+ }
+
+ // FIXME: Scan modules asynchronously here as well.
+
+ SrcLocOffset++;
+ SmallVector<IdentifierLoc, 2> Path;
+ IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
+ Path.emplace_back(IDLocation, ModuleID);
+ auto ModResult = CI.loadModule(IDLocation, Path, Module::Hidden, false);
+
+ assert(CB && "Must have PPCallbacks after module loading");
+ CB->moduleImport(SourceLocation(), Path, ModResult);
+
+ if (!ModResult)
+ return false;
+
+ if (CI.getDiagnostics().hasErrorOccurred())
+ return false;
+
+ MDC->run(Consumer);
+ return applyAndReport(*MDC, ModuleInvocation, Consumer, Controller,
+ CommandLine[0]);
+ }
+
+ std::shared_ptr<ModuleDepCollector>
+ scanTranslationUnit(DependencyConsumer &Consumer,
+ DependencyActionController &Controller) {
+ assert(CIPtr && "CIPtr must be initialized before calling this method");
+ auto &CI = *CIPtr;
+
+ std::optional<AsyncModuleCompiles> AsyncCompiles;
+ if (Worker.Service.getOpts().AsyncScanModules) {
+ AsyncCompiles.emplace();
+ if (!prescanModulesAsync(*AsyncCompiles, Controller))
+ return nullptr;
+ }
+
+ auto MDC = initializeScanInstanceDependencyCollector(
+ CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
+ Worker.Service, *OriginalInvocation, Controller, PrebuiltModuleASTMap,
+ StableDirs);
+
+ if (CI.getDiagnostics().hasErrorOccurred())
+ return nullptr;
+
+ if (!Controller.initialize(CI, *OriginalInvocation))
+ return nullptr;
+
+ ReadPCHAndPreprocessAction Action;
+ if (!CI.ExecuteAction(Action))
+ return nullptr;
+
+ MDC->run(Consumer);
+ if (!applyAndReport(*MDC, *OriginalInvocation, Consumer, Controller,
+ CommandLine[0]))
+ return nullptr;
+ return MDC;
+ }
+
+ bool applyAndReport(ModuleDepCollector &MDC,
+ CompilerInvocation &ModuleInvocation,
+ DependencyConsumer &Consumer,
+ DependencyActionController &Controller,
+ StringRef Executable) {
+ MDC.applyDiscoveredDependencies(ModuleInvocation);
+
+ bool Success = ModuleInvocation.withCowRef<bool>(
+ [&](CowCompilerInvocation &CowModuleInvocation) {
+ return Controller.finalize(*CIPtr, CowModuleInvocation);
+ });
+ if (!Success)
+ return false;
+
+ Consumer.handleBuildCommand(
+ {Executable.str(), ModuleInvocation.getCC1CommandLine()});
+ return true;
+ }
+};
+} // namespace dependencies
+} // namespace clang
+
DependencyScanningWorker::DependencyScanningWorker(
DependencyScanningService &Service)
: Service(Service) {
diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp
index 057e9dbfa2340..3d1c0bac1a929 100644
--- a/clang/lib/Tooling/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanningTool.cpp
@@ -9,7 +9,6 @@
#include "clang/Tooling/DependencyScanningTool.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticFrontend.h"
-#include "clang/DependencyScanning/CompilerInstanceWithContext.h"
#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
@@ -287,7 +286,7 @@ initVFSForByNameScanning(ArrayRef<std::string> CommandLine) {
// locations for the diagnostics. Therefore, sharing this global buffer across
// threads is ok.
static const std::string FakeInput(
- dependencies::CompilerInstanceWithContext::MaxNumOfQueries, ' ');
+ dependencies::DependencyScanningWorker::MaxNumOfByNameQueries, ' ');
StringRef InputPath =
llvm::sys::path::is_style_windows(llvm::sys::path::Style::native)
>From ca180ccb3253faa5fc10360a0fe6d9ca0a6c0408 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Mon, 6 Jul 2026 16:03:31 -0700
Subject: [PATCH 13/13] 2. Fold DependencyScannerImpl into
DependencyScanningWorker.cpp and DependencyScannerTool.cpp.
---
.../DependencyScannerImpl.h | 139 ----
.../DependencyScanningUtils.h | 1 -
.../DependencyScanningWorker.h | 17 +-
.../clang/Tooling/DependencyScanningTool.h | 1 -
clang/lib/DependencyScanning/CMakeLists.txt | 1 -
.../DependencyScannerImpl.cpp | 672 -----------------
.../DependencyScanningWorker.cpp | 680 +++++++++++++++++-
clang/lib/Tooling/DependencyScanningTool.cpp | 16 +-
8 files changed, 709 insertions(+), 818 deletions(-)
delete mode 100644 clang/include/clang/DependencyScanning/DependencyScannerImpl.h
delete mode 100644 clang/lib/DependencyScanning/DependencyScannerImpl.cpp
diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
deleted file mode 100644
index dea2cf5bffc4f..0000000000000
--- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
+++ /dev/null
@@ -1,139 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNERIMPL_H
-#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNERIMPL_H
-
-#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
-#include "clang/DependencyScanning/ModuleDepCollector.h"
-#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Frontend/CompilerInvocation.h"
-#include "clang/Frontend/TextDiagnosticPrinter.h"
-#include "llvm/Support/VirtualFileSystem.h"
-
-#include <mutex>
-#include <thread>
-
-namespace clang {
-class DiagnosticConsumer;
-
-namespace dependencies {
-class DependencyScanningService;
-class DependencyScanningWorker;
-
-class DependencyActionController;
-class DependencyScanningWorkerFilesystem;
-
-// Helper functions and data types.
-std::unique_ptr<DiagnosticOptions>
-createDiagOptions(ArrayRef<std::string> CommandLine);
-
-struct DiagnosticsEngineWithDiagOpts {
- // We need to bound the lifetime of the DiagOpts used to create the
- // DiganosticsEngine with the DiagnosticsEngine itself.
- std::unique_ptr<DiagnosticOptions> DiagOpts;
- IntrusiveRefCntPtr<DiagnosticsEngine> DiagEngine;
-
- DiagnosticsEngineWithDiagOpts(ArrayRef<std::string> CommandLine,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- DiagnosticConsumer &DC);
-};
-
-struct TextDiagnosticsPrinterWithOutput {
- // We need to bound the lifetime of the data that supports the DiagPrinter
- // with it together so they have the same lifetime.
- std::string DiagnosticOutput;
- llvm::raw_string_ostream DiagnosticsOS;
- std::unique_ptr<DiagnosticOptions> DiagOpts;
- TextDiagnosticPrinter DiagPrinter;
-
- TextDiagnosticsPrinterWithOutput(ArrayRef<std::string> CommandLine)
- : DiagnosticsOS(DiagnosticOutput),
- DiagOpts(createDiagOptions(CommandLine)),
- DiagPrinter(DiagnosticsOS, *DiagOpts) {}
-};
-
-/// Manages (and terminates) the asynchronous compilation of modules.
-class AsyncModuleCompiles {
- std::mutex Mutex;
- bool Stop = false;
- // FIXME: Have the service own a thread pool and use that instead.
- std::vector<std::thread> Compiles;
-
-public:
- /// Registers the module compilation, unless this instance is about to be
- /// destroyed.
- void add(llvm::unique_function<void()> Compile) {
- std::lock_guard<std::mutex> Lock(Mutex);
- if (!Stop)
- Compiles.emplace_back(std::move(Compile));
- }
-
- ~AsyncModuleCompiles() {
- {
- // Prevent registration of further module compiles.
- std::lock_guard<std::mutex> Lock(Mutex);
- Stop = true;
- }
-
- // Wait for outstanding module compiles to finish.
- for (std::thread &Compile : Compiles)
- Compile.join();
- }
-};
-
-void runTUModulePrescan(CompilerInstance &PrescanCI,
- DependencyScanningService &Service,
- DependencyActionController &Controller,
- AsyncModuleCompiles &Compiles);
-
-std::unique_ptr<CompilerInvocation>
-createCompilerInvocation(ArrayRef<std::string> CommandLine,
- DiagnosticsEngine &Diags);
-
-/// Canonicalizes command-line macro defines (e.g. removing "-DX -UX").
-void canonicalizeDefines(PreprocessorOptions &PPOpts);
-
-/// Creates a CompilerInvocation suitable for the dependency scanner.
-std::shared_ptr<CompilerInvocation>
-createScanCompilerInvocation(const CompilerInvocation &Invocation,
- const DependencyScanningService &Service,
- DependencyActionController &Controller);
-
-/// Creates dependency output options to be reported to the dependency consumer,
-/// deducing missing information if necessary.
-std::unique_ptr<DependencyOutputOptions>
-createDependencyOutputOptions(const CompilerInvocation &Invocation);
-
-void initializeScanCompilerInstance(
- CompilerInstance &ScanInstance,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
- IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS);
-
-SmallVector<StringRef>
-getInitialStableDirs(const CompilerInstance &ScanInstance);
-
-std::optional<PrebuiltModulesAttrsMap>
-computePrebuiltModulesASTMap(CompilerInstance &ScanInstance,
- SmallVector<StringRef> &StableDirs);
-
-/// Create the dependency collector that will collect the produced
-/// dependencies. May return the created ModuleDepCollector depending
-/// on the scanning format.
-std::shared_ptr<ModuleDepCollector> initializeScanInstanceDependencyCollector(
- CompilerInstance &ScanInstance,
- std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
- DependencyScanningService &Service, CompilerInvocation &Inv,
- DependencyActionController &Controller,
- PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
- SmallVector<StringRef> &StableDirs);
-} // namespace dependencies
-} // namespace clang
-
-#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNERIMPL_H
diff --git a/clang/include/clang/DependencyScanning/DependencyScanningUtils.h b/clang/include/clang/DependencyScanning/DependencyScanningUtils.h
index 33f7216762b03..952342d2a322d 100644
--- a/clang/include/clang/DependencyScanning/DependencyScanningUtils.h
+++ b/clang/include/clang/DependencyScanning/DependencyScanningUtils.h
@@ -11,7 +11,6 @@
#include "clang/DependencyScanning/DependencyActionController.h"
#include "clang/DependencyScanning/DependencyConsumer.h"
-#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/DependencyScanning/DependencyScanningWorker.h"
#include "clang/DependencyScanning/ModuleDepCollector.h"
#include "llvm/ADT/DenseSet.h"
diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
index ecf6d97f7e847..80eec488c30dc 100644
--- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
@@ -9,10 +9,10 @@
#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
+#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/LLVM.h"
-#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/DependencyScanning/DependencyScanningService.h"
#include "clang/DependencyScanning/ModuleDepCollector.h"
#include "clang/Frontend/PCHContainerOperations.h"
@@ -31,7 +31,20 @@ namespace dependencies {
class DependencyConsumer;
class DependencyScanningWorkerFilesystem;
-class CompilerInstanceWithContext;
+
+std::unique_ptr<DiagnosticOptions>
+createDiagOptions(ArrayRef<std::string> CommandLine);
+
+struct DiagnosticsEngineWithDiagOpts {
+ // We need to bound the lifetime of the DiagOpts used to create the
+ // DiganosticsEngine with the DiagnosticsEngine itself.
+ std::unique_ptr<DiagnosticOptions> DiagOpts;
+ IntrusiveRefCntPtr<DiagnosticsEngine> DiagEngine;
+
+ DiagnosticsEngineWithDiagOpts(ArrayRef<std::string> CommandLine,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+ DiagnosticConsumer &DC);
+};
/// An individual dependency scanning worker that is able to run on its own
/// thread.
diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h
index c52b0330acae2..dffd6fc157343 100644
--- a/clang/include/clang/Tooling/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanningTool.h
@@ -9,7 +9,6 @@
#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
-#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/DependencyScanning/DependencyScanningService.h"
#include "clang/DependencyScanning/DependencyScanningUtils.h"
#include "clang/DependencyScanning/DependencyScanningWorker.h"
diff --git a/clang/lib/DependencyScanning/CMakeLists.txt b/clang/lib/DependencyScanning/CMakeLists.txt
index 015c6bcb12326..88eb6408842f9 100644
--- a/clang/lib/DependencyScanning/CMakeLists.txt
+++ b/clang/lib/DependencyScanning/CMakeLists.txt
@@ -11,7 +11,6 @@ add_clang_library(clangDependencyScanning
DependencyScanningService.cpp
DependencyScanningWorker.cpp
DependencyScanningUtils.cpp
- DependencyScannerImpl.cpp
InProcessModuleCache.cpp
ModuleDepCollector.cpp
diff --git a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
deleted file mode 100644
index 58c0deac76fa5..0000000000000
--- a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
+++ /dev/null
@@ -1,672 +0,0 @@
-//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/DependencyScanning/DependencyScannerImpl.h"
-#include "clang/Basic/DiagnosticFrontend.h"
-#include "clang/Basic/DiagnosticSerialization.h"
-#include "clang/DependencyScanning/DependencyActionController.h"
-#include "clang/DependencyScanning/DependencyConsumer.h"
-#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
-#include "clang/DependencyScanning/DependencyScanningService.h"
-#include "clang/DependencyScanning/DependencyScanningWorker.h"
-#include "clang/Frontend/FrontendActions.h"
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/Option/Option.h"
-#include "llvm/Support/AdvisoryLock.h"
-#include "llvm/Support/CrashRecoveryContext.h"
-#include "llvm/Support/VirtualFileSystem.h"
-#include "llvm/TargetParser/Host.h"
-
-#include <mutex>
-#include <thread>
-
-using namespace clang;
-using namespace dependencies;
-
-static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
- const HeaderSearchOptions &ExistingHSOpts,
- DiagnosticsEngine *Diags,
- const LangOptions &LangOpts) {
- if (LangOpts.Modules) {
- if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
- if (Diags) {
- Diags->Report(diag::warn_pch_vfsoverlay_mismatch);
- auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
- if (VFSOverlays.empty()) {
- Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;
- } else {
- std::string Files = llvm::join(VFSOverlays, "\n");
- Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;
- }
- };
- VFSNote(0, HSOpts.VFSOverlayFiles);
- VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
- }
- }
- }
- return false;
-}
-
-namespace {
-
-using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
-
-/// A listener that collects the imported modules and the input
-/// files. While visiting, collect vfsoverlays and file inputs that determine
-/// whether prebuilt modules fully resolve in stable directories.
-class PrebuiltModuleListener : public ASTReaderListener {
-public:
- PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
- llvm::SmallVector<std::string> &NewModuleFiles,
- PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
- const HeaderSearchOptions &HSOpts,
- const LangOptions &LangOpts, DiagnosticsEngine &Diags,
- const ArrayRef<StringRef> StableDirs)
- : PrebuiltModuleFiles(PrebuiltModuleFiles),
- NewModuleFiles(NewModuleFiles),
- PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
- ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
-
- bool needsImportVisitation() const override { return true; }
- bool needsInputFileVisitation() override { return true; }
- bool needsSystemInputFileVisitation() override { return true; }
-
- /// Accumulate the modules are transitively depended on by the initial
- /// prebuilt module.
- void visitImport(StringRef ModuleName, StringRef Filename) override {
- if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
- NewModuleFiles.push_back(Filename.str());
-
- auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename);
- PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
- if (PrebuiltMapEntry.second)
- PrebuiltModule.setInStableDir(!StableDirs.empty());
-
- if (auto It = PrebuiltModulesASTMap.find(CurrentFile);
- It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
- PrebuiltModule.addDependent(It->getKey());
- }
-
- /// For each input file discovered, check whether it's external path is in a
- /// stable directory. Traversal is stopped if the current module is not
- /// considered stable.
- bool visitInputFileAsRequested(StringRef FilenameAsRequested,
- StringRef Filename, bool isSystem,
- bool isOverridden, time_t StoredTime,
- bool isExplicitModule) override {
- if (StableDirs.empty())
- return false;
- auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
- if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
- (!PrebuiltEntryIt->second.isInStableDir()))
- return false;
-
- PrebuiltEntryIt->second.setInStableDir(
- isPathInStableDir(StableDirs, Filename));
- return PrebuiltEntryIt->second.isInStableDir();
- }
-
- /// Update which module that is being actively traversed.
- void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind,
- bool DirectlyImported) override {
- // If the CurrentFile is not
- // considered stable, update any of it's transitive dependents.
- auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
- if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
- !PrebuiltEntryIt->second.isInStableDir())
- PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
- PrebuiltModulesASTMap);
- CurrentFile = Filename.str();
- }
-
- /// Check the header search options for a given module when considering
- /// if the module comes from stable directories.
- bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
- StringRef ModuleFilename, StringRef ContextHash,
- bool Complain) override {
-
- auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
- PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
- if (PrebuiltMapEntry.second)
- PrebuiltModule.setInStableDir(!StableDirs.empty());
-
- if (PrebuiltModule.isInStableDir())
- PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts));
-
- return false;
- }
-
- /// Accumulate vfsoverlays used to build these prebuilt modules.
- bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
- bool Complain) override {
-
- auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
- PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
- if (PrebuiltMapEntry.second)
- PrebuiltModule.setInStableDir(!StableDirs.empty());
-
- PrebuiltModule.setVFS(
- llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
-
- return checkHeaderSearchPaths(
- HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
- }
-
-private:
- PrebuiltModuleFilesT &PrebuiltModuleFiles;
- llvm::SmallVector<std::string> &NewModuleFiles;
- PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
- const HeaderSearchOptions &ExistingHSOpts;
- const LangOptions &ExistingLangOpts;
- DiagnosticsEngine &Diags;
- std::string CurrentFile;
- const ArrayRef<StringRef> StableDirs;
-};
-
-/// Visit the given prebuilt module and collect all of the modules it
-/// transitively imports and contributing input files.
-static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
- CompilerInstance &CI,
- PrebuiltModuleFilesT &ModuleFiles,
- PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
- DiagnosticsEngine &Diags,
- const ArrayRef<StringRef> StableDirs) {
- // List of module files to be processed.
- llvm::SmallVector<std::string> Worklist;
-
- PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
- CI.getHeaderSearchOpts(), CI.getLangOpts(),
- Diags, StableDirs);
-
- Listener.visitModuleFile(ModuleFileName::makeExplicit(PrebuiltModuleFilename),
- serialization::MK_ExplicitModule,
- /*DirectlyImported=*/true);
- if (ASTReader::readASTFileControlBlock(
- PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),
- CI.getPCHContainerReader(),
- /*FindModuleFileExtensions=*/false, Listener,
- /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))
- return true;
-
- while (!Worklist.empty()) {
- // FIXME: This is assuming the PCH only refers to explicitly-built modules,
- // which technically is not guaranteed. To remove the assumption, we'd need
- // to also rework how the module files are handled to the scan, specifically
- // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain
- // paths to ModuleFileName.
- Listener.visitModuleFile(ModuleFileName::makeExplicit(Worklist.back()),
- serialization::MK_ExplicitModule,
- /*DirectlyImported=*/false);
- if (ASTReader::readASTFileControlBlock(
- Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
- CI.getPCHContainerReader(),
- /*FindModuleFileExtensions=*/false, Listener,
- /*ValidateDiagnosticOptions=*/false))
- return true;
- }
- return false;
-}
-
-/// Transform arbitrary file name into an object-like file name.
-static std::string makeObjFileName(StringRef FileName) {
- SmallString<128> ObjFileName(FileName);
- llvm::sys::path::replace_extension(ObjFileName, "o");
- return std::string(ObjFileName);
-}
-
-/// Deduce the dependency target based on the output file and input files.
-static std::string
-deduceDepTarget(const std::string &OutputFile,
- const SmallVectorImpl<FrontendInputFile> &InputFiles) {
- if (OutputFile != "-")
- return OutputFile;
-
- if (InputFiles.empty() || !InputFiles.front().isFile())
- return "clang-scan-deps\\ dependency";
-
- return makeObjFileName(InputFiles.front().getFile());
-}
-
-// Clang implements -D and -U by splatting text into a predefines buffer. This
-// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
-// define the same macro, or adding C++ style comments before the macro name.
-//
-// This function checks that the first non-space characters in the macro
-// obviously form an identifier that can be uniqued on without lexing. Failing
-// to do this could lead to changing the final definition of a macro.
-//
-// We could set up a preprocessor and actually lex the name, but that's very
-// heavyweight for a situation that will almost never happen in practice.
-static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
- StringRef Name = Macro.split("=").first.ltrim(" \t");
- std::size_t I = 0;
-
- auto FinishName = [&]() -> std::optional<StringRef> {
- StringRef SimpleName = Name.slice(0, I);
- if (SimpleName.empty())
- return std::nullopt;
- return SimpleName;
- };
-
- for (; I != Name.size(); ++I) {
- switch (Name[I]) {
- case '(': // Start of macro parameter list
- case ' ': // End of macro name
- case '\t':
- return FinishName();
- case '_':
- continue;
- default:
- if (llvm::isAlnum(Name[I]))
- continue;
- return std::nullopt;
- }
- }
- return FinishName();
-}
-} // namespace
-
-void dependencies::canonicalizeDefines(PreprocessorOptions &PPOpts) {
- using MacroOpt = std::pair<StringRef, std::size_t>;
- std::vector<MacroOpt> SimpleNames;
- SimpleNames.reserve(PPOpts.Macros.size());
- std::size_t Index = 0;
- for (const auto &M : PPOpts.Macros) {
- auto SName = getSimpleMacroName(M.first);
- // Skip optimizing if we can't guarantee we can preserve relative order.
- if (!SName)
- return;
- SimpleNames.emplace_back(*SName, Index);
- ++Index;
- }
-
- llvm::stable_sort(SimpleNames, llvm::less_first());
- // Keep the last instance of each macro name by going in reverse
- auto NewEnd = std::unique(
- SimpleNames.rbegin(), SimpleNames.rend(),
- [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
- SimpleNames.erase(SimpleNames.begin(), NewEnd.base());
-
- // Apply permutation.
- decltype(PPOpts.Macros) NewMacros;
- NewMacros.reserve(SimpleNames.size());
- for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
- std::size_t OriginalIndex = SimpleNames[I].second;
- // We still emit undefines here as they may be undefining a predefined macro
- NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex]));
- }
- std::swap(PPOpts.Macros, NewMacros);
-}
-
-namespace {
-class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
- DependencyScanningWorkerFilesystem *DepFS;
-
-public:
- ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
- FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) {
- auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS);
- if (DFS) {
- assert(!DepFS && "Found multiple scanning VFSs");
- DepFS = DFS;
- }
- });
- assert(DepFS && "Did not find scanning VFS");
- }
-
- std::unique_ptr<DependencyDirectivesGetter>
- cloneFor(FileManager &FileMgr) override {
- return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr);
- }
-
- std::optional<ArrayRef<dependency_directives_scan::Directive>>
- operator()(FileEntryRef File) override {
- return DepFS->getDirectiveTokens(File.getName());
- }
-};
-
-/// Sanitize diagnostic options for dependency scan.
-void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
- // Don't print 'X warnings and Y errors generated'.
- DiagOpts.ShowCarets = false;
- // Don't write out diagnostic file.
- DiagOpts.DiagnosticSerializationFile.clear();
- // Don't emit warnings except for scanning specific warnings.
- // TODO: It would be useful to add a more principled way to ignore all
- // warnings that come from source code. The issue is that we need to
- // ignore warnings that could be surpressed by
- // `#pragma clang diagnostic`, while still allowing some scanning
- // warnings for things we're not ready to turn into errors yet.
- // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
- llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) {
- return llvm::StringSwitch<bool>(Warning)
- .Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false)
- .StartsWith("no-error=", false)
- .Default(true);
- });
-}
-} // namespace
-
-std::unique_ptr<DiagnosticOptions>
-dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {
- std::vector<const char *> CLI;
- for (const std::string &Arg : CommandLine)
- CLI.push_back(Arg.c_str());
- auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
- sanitizeDiagOpts(*DiagOpts);
- return DiagOpts;
-}
-
-DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
- ArrayRef<std::string> CommandLine,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
- std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
- llvm::transform(CommandLine, CCommandLine.begin(),
- [](const std::string &Str) { return Str.c_str(); });
- DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);
- sanitizeDiagOpts(*DiagOpts);
- DiagEngine = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC,
- /*ShouldOwnClient=*/false);
-}
-
-std::unique_ptr<CompilerInvocation>
-dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,
- DiagnosticsEngine &Diags) {
- llvm::opt::ArgStringList Argv;
- for (const std::string &Str : ArrayRef(CommandLine).drop_front())
- Argv.push_back(Str.c_str());
-
- auto Invocation = std::make_unique<CompilerInvocation>();
- if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) {
- // FIXME: Should we just go on like cc1_main does?
- return nullptr;
- }
- return Invocation;
-}
-
-void dependencies::initializeScanCompilerInstance(
- CompilerInstance &ScanInstance,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
- IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
- ScanInstance.setBuildingModule(false);
- ScanInstance.createVirtualFileSystem(FS, DiagConsumer);
- ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
- if (!Service.getOpts().EmitWarnings)
- ScanInstance.getDiagnostics().setIgnoreAllWarnings(true);
- ScanInstance.createFileManager();
- ScanInstance.createSourceManager();
-
- // Use DepFS for getting the dependency directives if requested to do so.
- if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan)
- ScanInstance.setDependencyDirectivesGetter(
- std::make_unique<ScanningDependencyDirectivesGetter>(
- ScanInstance.getFileManager()));
-}
-
-std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation(
- const CompilerInvocation &Invocation,
- const DependencyScanningService &Service,
- DependencyActionController &Controller) {
- auto ScanInvocation = std::make_shared<CompilerInvocation>(Invocation);
-
- sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts());
-
- ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
- true;
-
- if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
- ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
- Service.getOpts().BuildSessionTimestamp;
-
- ScanInvocation->getFrontendOpts().DisableFree = false;
- ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
- ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
- ScanInvocation->getFrontendOpts().GenReducedBMI = false;
- ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
- // This will prevent us compiling individual modules asynchronously since
- // FileManager is not thread-safe, but it does improve performance for now.
- ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
- ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
- ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
- any(Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
-
- // Consider different header search and diagnostic options to create
- // different modules. This avoids the unsound aliasing of module PCMs.
- //
- // TODO: Implement diagnostic bucketing to reduce the impact of strict
- // context hashing.
- ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
- ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
- ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
- ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
- ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
- true;
- ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
-
- // Application extension only affects the handling of availability attributes,
- // which cannot change the dependencies.
- ScanInvocation->getLangOpts().AppExt = false;
-
- // Ensure that the scanner does not create new dependency collectors,
- // and thus won't write out the extra '.d' files to disk.
- ScanInvocation->getDependencyOutputOpts() = {};
-
- Controller.initializeScanInvocation(*ScanInvocation);
-
- return ScanInvocation;
-}
-
-llvm::SmallVector<StringRef>
-dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) {
- // Create a collection of stable directories derived from the ScanInstance
- // for determining whether module dependencies would fully resolve from
- // those directories.
- llvm::SmallVector<StringRef> StableDirs;
- const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
- if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot))
- StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
- return StableDirs;
-}
-
-std::optional<PrebuiltModulesAttrsMap>
-dependencies::computePrebuiltModulesASTMap(
- CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {
- // Store a mapping of prebuilt module files and their properties like header
- // search options. This will prevent the implicit build to create duplicate
- // modules and will force reuse of the existing prebuilt module files
- // instead.
- PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
-
- if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
- if (visitPrebuiltModule(
- ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
- ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
- PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs))
- return {};
-
- return PrebuiltModulesASTMap;
-}
-
-std::unique_ptr<DependencyOutputOptions>
-dependencies::createDependencyOutputOptions(
- const CompilerInvocation &Invocation) {
- auto Opts = std::make_unique<DependencyOutputOptions>(
- Invocation.getDependencyOutputOpts());
- // We need at least one -MT equivalent for the generator of make dependency
- // files to work.
- if (Opts->Targets.empty())
- Opts->Targets = {deduceDepTarget(Invocation.getFrontendOpts().OutputFile,
- Invocation.getFrontendOpts().Inputs)};
- Opts->IncludeSystemHeaders = true;
-
- return Opts;
-}
-
-std::shared_ptr<ModuleDepCollector>
-dependencies::initializeScanInstanceDependencyCollector(
- CompilerInstance &ScanInstance,
- std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
- DependencyScanningService &Service, CompilerInvocation &Inv,
- DependencyActionController &Controller,
- PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
- SmallVector<StringRef> &StableDirs) {
- auto MDC = std::make_shared<ModuleDepCollector>(
- Service, std::move(DepOutputOpts), ScanInstance, Controller, Inv,
- std::move(PrebuiltModulesASTMap), StableDirs);
- ScanInstance.addDependencyCollector(MDC);
- return MDC;
-}
-
-struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
- DependencyScanningService &Service;
- DependencyActionController &Controller;
- AsyncModuleCompiles &Compiles;
-
- SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
- DependencyActionController &Controller,
- AsyncModuleCompiles &Compiles)
- : Service(Service), Controller(Controller), Compiles(Compiles) {}
-
- bool BeginSourceFileAction(CompilerInstance &CI) override;
-};
-
-/// The preprocessor callback that takes care of initiating an asynchronous
-/// module compilation if needed.
-struct AsyncModuleCompile : PPCallbacks {
- CompilerInstance &CI;
- DependencyScanningService &Service;
- DependencyActionController &Controller;
- AsyncModuleCompiles &Compiles;
-
- AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
- DependencyActionController &Controller,
- AsyncModuleCompiles &Compiles)
- : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
-
- void moduleLoadSkipped(Module *M) override {
- M = M->getTopLevelModule();
-
- HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
- ModuleCache &ModCache = CI.getModuleCache();
- ModuleFileName ModuleFileName = HS.getCachedModuleFileName(M);
-
- uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName);
- // Someone else already built/validated the PCM.
- if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
- return;
-
- if (!CI.getASTReader())
- CI.createASTReader();
- SmallVector<ASTReader::ImportedModule, 0> Imported;
- // Only calling ReadASTCore() to avoid the expensive eager deserialization
- // of the clang::Module objects in ReadAST().
- // FIXME: Consider doing this in the new thread depending on how expensive
- // the read turns out to be.
- switch (CI.getASTReader()->ReadASTCore(
- ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(),
- nullptr, Imported, {}, {}, {},
- ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
- ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
- case ASTReader::Success:
- // We successfully read a valid, up-to-date PCM.
- // FIXME: This could update the timestamp. Regular calls to
- // ASTReader::ReadAST() would do so unless they encountered corrupted
- // AST block, corrupted extension block, or did not read the expected
- // top-level module.
- return;
- case ASTReader::OutOfDate:
- case ASTReader::Missing:
- // The most interesting case.
- break;
- default:
- // Let the regular scan diagnose this.
- return;
- }
-
- auto Lock = ModCache.getLock(ModuleFileName);
- bool Owned;
- llvm::Error LockErr = Lock->tryLock().moveInto(Owned);
- // Someone else is building the PCM right now.
- if (!LockErr && !Owned)
- return;
- // We should build the PCM.
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
- llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
- Service, Service.getOpts().MakeVFS());
- VFS = createVFSFromCompilerInvocation(CI.getInvocation(),
- CI.getDiagnostics(), std::move(VFS));
- auto DC = std::make_unique<DiagnosticConsumer>();
- auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries());
- CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
- std::move(MC));
- auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
- CloneConfig);
- auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
- CloneConfig);
-
- auto ModController = Controller.clone();
-
- // Note: This lock belongs to a module cache that might not outlive the
- // thread. This works, because the in-process lock only refers to an object
- // managed by the service, which does outlive the thread.
- Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
- ModCI2 = std::move(ModCI2), DC = std::move(DC),
- ModController = std::move(ModController), Service = &Service,
- Compiles = &Compiles] {
- llvm::CrashRecoveryContext CRC;
- (void)CRC.RunSafely([&] {
- // Quickly discovers and compiles modules for the real scan below.
- SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
- *Compiles);
- (void)ModCI1->ExecuteAction(Action1);
- // The real scan below.
- ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
- GenerateModuleFromModuleMapAction Action2;
- (void)ModCI2->ExecuteAction(Action2);
- });
- });
- }
-};
-
-/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
-/// modules asynchronously without blocking or importing them.
-struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
- DependencyScanningService &Service;
- DependencyActionController &Controller;
- AsyncModuleCompiles &Compiles;
-
- SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
- DependencyActionController &Controller,
- AsyncModuleCompiles &Compiles)
- : Service(Service), Controller(Controller), Compiles(Compiles) {}
-
- bool BeginSourceFileAction(CompilerInstance &CI) override {
- CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
- CI.getPreprocessor().addPPCallbacks(std::make_unique<AsyncModuleCompile>(
- CI, Service, Controller, Compiles));
- return true;
- }
-};
-
-bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
- CompilerInstance &CI) {
- CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
- CI.getPreprocessor().addPPCallbacks(
- std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles));
- return true;
-}
-
-void dependencies::runTUModulePrescan(CompilerInstance &PrescanCI,
- DependencyScanningService &Service,
- DependencyActionController &Controller,
- AsyncModuleCompiles &Compiles) {
- SingleTUWithAsyncModuleCompiles Action(Service, Controller, Compiles);
- (void)PrescanCI.ExecuteAction(Action);
-}
diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
index 853caf48262e4..343a7913a79bb 100644
--- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
@@ -9,9 +9,9 @@
#include "clang/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticFrontend.h"
+#include "clang/Basic/DiagnosticSerialization.h"
#include "clang/DependencyScanning/DependencyActionController.h"
#include "clang/DependencyScanning/DependencyConsumer.h"
-#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/DependencyScanning/DependencyScanningUtils.h"
#include "clang/DependencyScanning/InProcessModuleCache.h"
#include "clang/Frontend/CompilerInvocation.h"
@@ -20,14 +20,692 @@
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Option/Option.h"
+#include "llvm/Support/AdvisoryLock.h"
+#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/VirtualFileSystem.h"
+#include <mutex>
#include <optional>
+#include <thread>
using namespace clang;
using namespace dependencies;
+namespace {
+bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
+ const HeaderSearchOptions &ExistingHSOpts,
+ DiagnosticsEngine *Diags,
+ const LangOptions &LangOpts) {
+ if (LangOpts.Modules) {
+ if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
+ if (Diags) {
+ Diags->Report(diag::warn_pch_vfsoverlay_mismatch);
+ auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
+ if (VFSOverlays.empty()) {
+ Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;
+ } else {
+ std::string Files = llvm::join(VFSOverlays, "\n");
+ Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;
+ }
+ };
+ VFSNote(0, HSOpts.VFSOverlayFiles);
+ VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
+ }
+ }
+ }
+ return false;
+}
+
+using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
+
+/// A listener that collects the imported modules and the input
+/// files. While visiting, collect vfsoverlays and file inputs that determine
+/// whether prebuilt modules fully resolve in stable directories.
+class PrebuiltModuleListener : public ASTReaderListener {
+public:
+ PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
+ llvm::SmallVector<std::string> &NewModuleFiles,
+ PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
+ const HeaderSearchOptions &HSOpts,
+ const LangOptions &LangOpts, DiagnosticsEngine &Diags,
+ const ArrayRef<StringRef> StableDirs)
+ : PrebuiltModuleFiles(PrebuiltModuleFiles),
+ NewModuleFiles(NewModuleFiles),
+ PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
+ ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
+
+ bool needsImportVisitation() const override { return true; }
+ bool needsInputFileVisitation() override { return true; }
+ bool needsSystemInputFileVisitation() override { return true; }
+
+ /// Accumulate the modules are transitively depended on by the initial
+ /// prebuilt module.
+ void visitImport(StringRef ModuleName, StringRef Filename) override {
+ if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
+ NewModuleFiles.push_back(Filename.str());
+
+ auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename);
+ PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
+ if (PrebuiltMapEntry.second)
+ PrebuiltModule.setInStableDir(!StableDirs.empty());
+
+ if (auto It = PrebuiltModulesASTMap.find(CurrentFile);
+ It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
+ PrebuiltModule.addDependent(It->getKey());
+ }
+
+ /// For each input file discovered, check whether it's external path is in a
+ /// stable directory. Traversal is stopped if the current module is not
+ /// considered stable.
+ bool visitInputFileAsRequested(StringRef FilenameAsRequested,
+ StringRef Filename, bool isSystem,
+ bool isOverridden, time_t StoredTime,
+ bool isExplicitModule) override {
+ if (StableDirs.empty())
+ return false;
+ auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
+ if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
+ (!PrebuiltEntryIt->second.isInStableDir()))
+ return false;
+
+ PrebuiltEntryIt->second.setInStableDir(
+ isPathInStableDir(StableDirs, Filename));
+ return PrebuiltEntryIt->second.isInStableDir();
+ }
+
+ /// Update which module that is being actively traversed.
+ void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind,
+ bool DirectlyImported) override {
+ // If the CurrentFile is not
+ // considered stable, update any of it's transitive dependents.
+ auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
+ if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
+ !PrebuiltEntryIt->second.isInStableDir())
+ PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
+ PrebuiltModulesASTMap);
+ CurrentFile = Filename.str();
+ }
+
+ /// Check the header search options for a given module when considering
+ /// if the module comes from stable directories.
+ bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
+ StringRef ModuleFilename, StringRef ContextHash,
+ bool Complain) override {
+
+ auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
+ PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
+ if (PrebuiltMapEntry.second)
+ PrebuiltModule.setInStableDir(!StableDirs.empty());
+
+ if (PrebuiltModule.isInStableDir())
+ PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts));
+
+ return false;
+ }
+
+ /// Accumulate vfsoverlays used to build these prebuilt modules.
+ bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
+ bool Complain) override {
+
+ auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
+ PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
+ if (PrebuiltMapEntry.second)
+ PrebuiltModule.setInStableDir(!StableDirs.empty());
+
+ PrebuiltModule.setVFS(
+ llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
+
+ return checkHeaderSearchPaths(
+ HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
+ }
+
+private:
+ PrebuiltModuleFilesT &PrebuiltModuleFiles;
+ llvm::SmallVector<std::string> &NewModuleFiles;
+ PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
+ const HeaderSearchOptions &ExistingHSOpts;
+ const LangOptions &ExistingLangOpts;
+ DiagnosticsEngine &Diags;
+ std::string CurrentFile;
+ const ArrayRef<StringRef> StableDirs;
+};
+
+/// Visit the given prebuilt module and collect all of the modules it
+/// transitively imports and contributing input files.
+bool visitPrebuiltModule(StringRef PrebuiltModuleFilename, CompilerInstance &CI,
+ PrebuiltModuleFilesT &ModuleFiles,
+ PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
+ DiagnosticsEngine &Diags,
+ const ArrayRef<StringRef> StableDirs) {
+ // List of module files to be processed.
+ llvm::SmallVector<std::string> Worklist;
+
+ PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
+ CI.getHeaderSearchOpts(), CI.getLangOpts(),
+ Diags, StableDirs);
+
+ Listener.visitModuleFile(ModuleFileName::makeExplicit(PrebuiltModuleFilename),
+ serialization::MK_ExplicitModule,
+ /*DirectlyImported=*/true);
+ if (ASTReader::readASTFileControlBlock(
+ PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),
+ CI.getPCHContainerReader(),
+ /*FindModuleFileExtensions=*/false, Listener,
+ /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))
+ return true;
+
+ while (!Worklist.empty()) {
+ // FIXME: This is assuming the PCH only refers to explicitly-built modules,
+ // which technically is not guaranteed. To remove the assumption, we'd need
+ // to also rework how the module files are handled to the scan, specifically
+ // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain
+ // paths to ModuleFileName.
+ Listener.visitModuleFile(ModuleFileName::makeExplicit(Worklist.back()),
+ serialization::MK_ExplicitModule,
+ /*DirectlyImported=*/false);
+ if (ASTReader::readASTFileControlBlock(
+ Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
+ CI.getPCHContainerReader(),
+ /*FindModuleFileExtensions=*/false, Listener,
+ /*ValidateDiagnosticOptions=*/false))
+ return true;
+ }
+ return false;
+}
+
+/// Transform arbitrary file name into an object-like file name.
+std::string makeObjFileName(StringRef FileName) {
+ SmallString<128> ObjFileName(FileName);
+ llvm::sys::path::replace_extension(ObjFileName, "o");
+ return std::string(ObjFileName);
+}
+
+/// Deduce the dependency target based on the output file and input files.
+std::string
+deduceDepTarget(const std::string &OutputFile,
+ const SmallVectorImpl<FrontendInputFile> &InputFiles) {
+ if (OutputFile != "-")
+ return OutputFile;
+
+ if (InputFiles.empty() || !InputFiles.front().isFile())
+ return "clang-scan-deps\\ dependency";
+
+ return makeObjFileName(InputFiles.front().getFile());
+}
+
+// Clang implements -D and -U by splatting text into a predefines buffer. This
+// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
+// define the same macro, or adding C++ style comments before the macro name.
+//
+// This function checks that the first non-space characters in the macro
+// obviously form an identifier that can be uniqued on without lexing. Failing
+// to do this could lead to changing the final definition of a macro.
+//
+// We could set up a preprocessor and actually lex the name, but that's very
+// heavyweight for a situation that will almost never happen in practice.
+std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
+ StringRef Name = Macro.split("=").first.ltrim(" \t");
+ std::size_t I = 0;
+
+ auto FinishName = [&]() -> std::optional<StringRef> {
+ StringRef SimpleName = Name.slice(0, I);
+ if (SimpleName.empty())
+ return std::nullopt;
+ return SimpleName;
+ };
+
+ for (; I != Name.size(); ++I) {
+ switch (Name[I]) {
+ case '(': // Start of macro parameter list
+ case ' ': // End of macro name
+ case '\t':
+ return FinishName();
+ case '_':
+ continue;
+ default:
+ if (llvm::isAlnum(Name[I]))
+ continue;
+ return std::nullopt;
+ }
+ }
+ return FinishName();
+}
+
+void canonicalizeDefines(PreprocessorOptions &PPOpts) {
+ using MacroOpt = std::pair<StringRef, std::size_t>;
+ std::vector<MacroOpt> SimpleNames;
+ SimpleNames.reserve(PPOpts.Macros.size());
+ std::size_t Index = 0;
+ for (const auto &M : PPOpts.Macros) {
+ auto SName = getSimpleMacroName(M.first);
+ // Skip optimizing if we can't guarantee we can preserve relative order.
+ if (!SName)
+ return;
+ SimpleNames.emplace_back(*SName, Index);
+ ++Index;
+ }
+
+ llvm::stable_sort(SimpleNames, llvm::less_first());
+ // Keep the last instance of each macro name by going in reverse
+ auto NewEnd = std::unique(
+ SimpleNames.rbegin(), SimpleNames.rend(),
+ [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
+ SimpleNames.erase(SimpleNames.begin(), NewEnd.base());
+
+ // Apply permutation.
+ decltype(PPOpts.Macros) NewMacros;
+ NewMacros.reserve(SimpleNames.size());
+ for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
+ std::size_t OriginalIndex = SimpleNames[I].second;
+ // We still emit undefines here as they may be undefining a predefined macro
+ NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex]));
+ }
+ std::swap(PPOpts.Macros, NewMacros);
+}
+
+std::unique_ptr<CompilerInvocation>
+createCompilerInvocation(ArrayRef<std::string> CommandLine,
+ DiagnosticsEngine &Diags) {
+ llvm::opt::ArgStringList Argv;
+ for (const std::string &Str : ArrayRef(CommandLine).drop_front())
+ Argv.push_back(Str.c_str());
+
+ auto Invocation = std::make_unique<CompilerInvocation>();
+ if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) {
+ // FIXME: Should we just go on like cc1_main does?
+ return nullptr;
+ }
+ return Invocation;
+}
+
+class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
+ DependencyScanningWorkerFilesystem *DepFS;
+
+public:
+ ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
+ FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) {
+ auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS);
+ if (DFS) {
+ assert(!DepFS && "Found multiple scanning VFSs");
+ DepFS = DFS;
+ }
+ });
+ assert(DepFS && "Did not find scanning VFS");
+ }
+
+ std::unique_ptr<DependencyDirectivesGetter>
+ cloneFor(FileManager &FileMgr) override {
+ return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr);
+ }
+
+ std::optional<ArrayRef<dependency_directives_scan::Directive>>
+ operator()(FileEntryRef File) override {
+ return DepFS->getDirectiveTokens(File.getName());
+ }
+};
+
+void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
+ // Don't print 'X warnings and Y errors generated'.
+ DiagOpts.ShowCarets = false;
+ // Don't write out diagnostic file.
+ DiagOpts.DiagnosticSerializationFile.clear();
+ // Don't emit warnings except for scanning specific warnings.
+ // TODO: It would be useful to add a more principled way to ignore all
+ // warnings that come from source code. The issue is that we need to
+ // ignore warnings that could be surpressed by
+ // `#pragma clang diagnostic`, while still allowing some scanning
+ // warnings for things we're not ready to turn into errors yet.
+ // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
+ llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) {
+ return llvm::StringSwitch<bool>(Warning)
+ .Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false)
+ .StartsWith("no-error=", false)
+ .Default(true);
+ });
+}
+
+/// Creates a CompilerInvocation suitable for the dependency scanner.
+std::shared_ptr<CompilerInvocation>
+createScanCompilerInvocation(const CompilerInvocation &Invocation,
+ const DependencyScanningService &Service,
+ DependencyActionController &Controller) {
+ auto ScanInvocation = std::make_shared<CompilerInvocation>(Invocation);
+
+ sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts());
+
+ ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
+ true;
+
+ if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
+ ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
+ Service.getOpts().BuildSessionTimestamp;
+
+ ScanInvocation->getFrontendOpts().DisableFree = false;
+ ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
+ ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
+ ScanInvocation->getFrontendOpts().GenReducedBMI = false;
+ ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
+ // This will prevent us compiling individual modules asynchronously since
+ // FileManager is not thread-safe, but it does improve performance for now.
+ ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
+ ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
+ ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
+ any(Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
+
+ // Consider different header search and diagnostic options to create
+ // different modules. This avoids the unsound aliasing of module PCMs.
+ //
+ // TODO: Implement diagnostic bucketing to reduce the impact of strict
+ // context hashing.
+ ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
+ ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
+ ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
+ ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
+ ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
+ true;
+ ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
+
+ // Application extension only affects the handling of availability attributes,
+ // which cannot change the dependencies.
+ ScanInvocation->getLangOpts().AppExt = false;
+
+ // Ensure that the scanner does not create new dependency collectors,
+ // and thus won't write out the extra '.d' files to disk.
+ ScanInvocation->getDependencyOutputOpts() = {};
+
+ Controller.initializeScanInvocation(*ScanInvocation);
+
+ return ScanInvocation;
+}
+
+/// Creates dependency output options to be reported to the dependency consumer,
+/// deducing missing information if necessary.
+std::unique_ptr<DependencyOutputOptions>
+createDependencyOutputOptions(const CompilerInvocation &Invocation) {
+ auto Opts = std::make_unique<DependencyOutputOptions>(
+ Invocation.getDependencyOutputOpts());
+ // We need at least one -MT equivalent for the generator of make dependency
+ // files to work.
+ if (Opts->Targets.empty())
+ Opts->Targets = {deduceDepTarget(Invocation.getFrontendOpts().OutputFile,
+ Invocation.getFrontendOpts().Inputs)};
+ Opts->IncludeSystemHeaders = true;
+
+ return Opts;
+}
+
+void initializeScanCompilerInstance(
+ CompilerInstance &ScanInstance,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+ DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
+ IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
+ ScanInstance.setBuildingModule(false);
+ ScanInstance.createVirtualFileSystem(FS, DiagConsumer);
+ ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
+ if (!Service.getOpts().EmitWarnings)
+ ScanInstance.getDiagnostics().setIgnoreAllWarnings(true);
+ ScanInstance.createFileManager();
+ ScanInstance.createSourceManager();
+
+ // Use DepFS for getting the dependency directives if requested to do so.
+ if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan)
+ ScanInstance.setDependencyDirectivesGetter(
+ std::make_unique<ScanningDependencyDirectivesGetter>(
+ ScanInstance.getFileManager()));
+}
+
+SmallVector<StringRef>
+getInitialStableDirs(const CompilerInstance &ScanInstance) {
+ // Create a collection of stable directories derived from the ScanInstance
+ // for determining whether module dependencies would fully resolve from
+ // those directories.
+ llvm::SmallVector<StringRef> StableDirs;
+ const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
+ if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot))
+ StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
+ return StableDirs;
+}
+
+std::optional<PrebuiltModulesAttrsMap>
+computePrebuiltModulesASTMap(CompilerInstance &ScanInstance,
+ llvm::SmallVector<StringRef> &StableDirs) {
+ // Store a mapping of prebuilt module files and their properties like header
+ // search options. This will prevent the implicit build to create duplicate
+ // modules and will force reuse of the existing prebuilt module files
+ // instead.
+ PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
+
+ if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
+ if (visitPrebuiltModule(
+ ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
+ ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
+ PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs))
+ return {};
+
+ return PrebuiltModulesASTMap;
+}
+
+/// Create the dependency collector that will collect the produced
+/// dependencies. May return the created ModuleDepCollector depending
+/// on the scanning format.
+std::shared_ptr<ModuleDepCollector> initializeScanInstanceDependencyCollector(
+ CompilerInstance &ScanInstance,
+ std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
+ DependencyScanningService &Service, CompilerInvocation &Inv,
+ DependencyActionController &Controller,
+ PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
+ SmallVector<StringRef> &StableDirs) {
+ auto MDC = std::make_shared<ModuleDepCollector>(
+ Service, std::move(DepOutputOpts), ScanInstance, Controller, Inv,
+ std::move(PrebuiltModulesASTMap), StableDirs);
+ ScanInstance.addDependencyCollector(MDC);
+ return MDC;
+}
+
+class AsyncModuleCompiles {
+ std::mutex Mutex;
+ bool Stop = false;
+ // FIXME: Have the service own a thread pool and use that instead.
+ std::vector<std::thread> Compiles;
+
+public:
+ /// Registers the module compilation, unless this instance is about to be
+ /// destroyed.
+ void add(llvm::unique_function<void()> Compile) {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ if (!Stop)
+ Compiles.emplace_back(std::move(Compile));
+ }
+
+ ~AsyncModuleCompiles() {
+ {
+ // Prevent registration of further module compiles.
+ std::lock_guard<std::mutex> Lock(Mutex);
+ Stop = true;
+ }
+
+ // Wait for outstanding module compiles to finish.
+ for (std::thread &Compile : Compiles)
+ Compile.join();
+ }
+};
+
+struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
+ DependencyScanningService &Service;
+ DependencyActionController &Controller;
+ AsyncModuleCompiles &Compiles;
+
+ SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles)
+ : Service(Service), Controller(Controller), Compiles(Compiles) {}
+
+ bool BeginSourceFileAction(CompilerInstance &CI) override;
+};
+
+/// The preprocessor callback that takes care of initiating an asynchronous
+/// module compilation if needed.
+struct AsyncModuleCompile : PPCallbacks {
+ CompilerInstance &CI;
+ DependencyScanningService &Service;
+ DependencyActionController &Controller;
+ AsyncModuleCompiles &Compiles;
+
+ AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles)
+ : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
+
+ void moduleLoadSkipped(Module *M) override {
+ M = M->getTopLevelModule();
+
+ HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
+ ModuleCache &ModCache = CI.getModuleCache();
+ ModuleFileName ModuleFileName = HS.getCachedModuleFileName(M);
+
+ uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName);
+ // Someone else already built/validated the PCM.
+ if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
+ return;
+
+ if (!CI.getASTReader())
+ CI.createASTReader();
+ SmallVector<ASTReader::ImportedModule, 0> Imported;
+ // Only calling ReadASTCore() to avoid the expensive eager deserialization
+ // of the clang::Module objects in ReadAST().
+ // FIXME: Consider doing this in the new thread depending on how expensive
+ // the read turns out to be.
+ switch (CI.getASTReader()->ReadASTCore(
+ ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(),
+ nullptr, Imported, {}, {}, {},
+ ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
+ ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
+ case ASTReader::Success:
+ // We successfully read a valid, up-to-date PCM.
+ // FIXME: This could update the timestamp. Regular calls to
+ // ASTReader::ReadAST() would do so unless they encountered corrupted
+ // AST block, corrupted extension block, or did not read the expected
+ // top-level module.
+ return;
+ case ASTReader::OutOfDate:
+ case ASTReader::Missing:
+ // The most interesting case.
+ break;
+ default:
+ // Let the regular scan diagnose this.
+ return;
+ }
+
+ auto Lock = ModCache.getLock(ModuleFileName);
+ bool Owned;
+ llvm::Error LockErr = Lock->tryLock().moveInto(Owned);
+ // Someone else is building the PCM right now.
+ if (!LockErr && !Owned)
+ return;
+ // We should build the PCM.
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
+ llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
+ Service, Service.getOpts().MakeVFS());
+ VFS = createVFSFromCompilerInvocation(CI.getInvocation(),
+ CI.getDiagnostics(), std::move(VFS));
+ auto DC = std::make_unique<DiagnosticConsumer>();
+ auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries());
+ CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
+ std::move(MC));
+ auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
+ CloneConfig);
+ auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
+ CloneConfig);
+
+ auto ModController = Controller.clone();
+
+ // Note: This lock belongs to a module cache that might not outlive the
+ // thread. This works, because the in-process lock only refers to an object
+ // managed by the service, which does outlive the thread.
+ Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
+ ModCI2 = std::move(ModCI2), DC = std::move(DC),
+ ModController = std::move(ModController), Service = &Service,
+ Compiles = &Compiles] {
+ llvm::CrashRecoveryContext CRC;
+ (void)CRC.RunSafely([&] {
+ // Quickly discovers and compiles modules for the real scan below.
+ SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
+ *Compiles);
+ (void)ModCI1->ExecuteAction(Action1);
+ // The real scan below.
+ ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
+ GenerateModuleFromModuleMapAction Action2;
+ (void)ModCI2->ExecuteAction(Action2);
+ });
+ });
+ }
+};
+
+/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
+/// modules asynchronously without blocking or importing them.
+struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
+ DependencyScanningService &Service;
+ DependencyActionController &Controller;
+ AsyncModuleCompiles &Compiles;
+
+ SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles)
+ : Service(Service), Controller(Controller), Compiles(Compiles) {}
+
+ bool BeginSourceFileAction(CompilerInstance &CI) override {
+ CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
+ CI.getPreprocessor().addPPCallbacks(std::make_unique<AsyncModuleCompile>(
+ CI, Service, Controller, Compiles));
+ return true;
+ }
+};
+
+bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
+ CompilerInstance &CI) {
+ CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
+ CI.getPreprocessor().addPPCallbacks(
+ std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles));
+ return true;
+}
+
+void runTUModulePrescan(CompilerInstance &PrescanCI,
+ DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles) {
+ SingleTUWithAsyncModuleCompiles Action(Service, Controller, Compiles);
+ (void)PrescanCI.ExecuteAction(Action);
+}
+} // namespace
+
namespace clang {
namespace dependencies {
+
+std::unique_ptr<DiagnosticOptions>
+createDiagOptions(ArrayRef<std::string> CommandLine) {
+ std::vector<const char *> CLI;
+ for (const std::string &Arg : CommandLine)
+ CLI.push_back(Arg.c_str());
+ auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
+ sanitizeDiagOpts(*DiagOpts);
+ return DiagOpts;
+}
+
+DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
+ ArrayRef<std::string> CommandLine,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
+ std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
+ llvm::transform(CommandLine, CCommandLine.begin(),
+ [](const std::string &Str) { return Str.c_str(); });
+ DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);
+ sanitizeDiagOpts(*DiagOpts);
+ DiagEngine = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC,
+ /*ShouldOwnClient=*/false);
+}
+
+// The CompilerInstanceWithContext (CIWC) is a friend of
+// DependencyScanningWorker to access the worker's private data, so it is
+// defined in the clang::dependencies namespace.
class CompilerInstanceWithContext {
// Context
DependencyScanningWorker &Worker;
diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp
index 3d1c0bac1a929..68c1ad6c988a8 100644
--- a/clang/lib/Tooling/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanningTool.cpp
@@ -9,11 +9,11 @@
#include "clang/Tooling/DependencyScanningTool.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticFrontend.h"
-#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/Tool.h"
#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/Lex/Preprocessor.h"
#include "llvm/ADT/ScopeExit.h"
@@ -86,6 +86,20 @@ class MakeDependencyPrinterConsumer : public DependencyConsumer {
std::vector<std::string> Dependencies;
std::vector<std::string> DependenciesFromModules;
};
+
+struct TextDiagnosticsPrinterWithOutput {
+ // We need to bound the lifetime of the data that supports the DiagPrinter
+ // with it together so they have the same lifetime.
+ std::string DiagnosticOutput;
+ llvm::raw_string_ostream DiagnosticsOS;
+ std::unique_ptr<DiagnosticOptions> DiagOpts;
+ TextDiagnosticPrinter DiagPrinter;
+
+ TextDiagnosticsPrinterWithOutput(ArrayRef<std::string> CommandLine)
+ : DiagnosticsOS(DiagnosticOutput),
+ DiagOpts(createDiagOptions(CommandLine)),
+ DiagPrinter(DiagnosticsOS, *DiagOpts) {}
+};
} // anonymous namespace
static std::pair<std::unique_ptr<driver::Driver>,
More information about the cfe-commits
mailing list