[clang] [clang][DependencyScanning] Use CompilerInstanceWithContext for TU Scanning (PR #211408)
Qiongsi Wu via cfe-commits
cfe-commits at lists.llvm.org
Fri Jul 31 09:16:25 PDT 2026
https://github.com/qiongsiwu updated https://github.com/llvm/llvm-project/pull/211408
>From b2b20dfb6c020c408e4466d1f2907f2362555754 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsi_wu at apple.com>
Date: Wed, 22 Jul 2026 11:38:54 -0700
Subject: [PATCH] Use CompilerInstanceWithContext for TU scanning.
---
.../DependencyScannerImpl.h | 27 --
.../DependencyScannerImpl.cpp | 293 ---------------
.../DependencyScanningWorker.cpp | 353 ++++++++++++++++--
clang/test/ClangScanDeps/logging-simple.c | 1 +
.../Tooling/DependencyScannerTest.cpp | 35 ++
5 files changed, 351 insertions(+), 358 deletions(-)
diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
index f973429a783c3..dff959a70be8f 100644
--- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
+++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
@@ -27,33 +27,6 @@ class DependencyConsumer;
class DependencyActionController;
class DependencyScanningWorkerFilesystem;
-class DependencyScanningAction {
-public:
- DependencyScanningAction(
- DependencyScanningService &Service, StringRef WorkingDirectory,
- DependencyConsumer &Consumer, DependencyActionController &Controller,
- IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS)
- : Service(Service), WorkingDirectory(WorkingDirectory),
- Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)) {}
- bool runInvocation(std::string Executable,
- std::unique_ptr<CompilerInvocation> Invocation,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- DiagnosticConsumer *DiagConsumer);
-
- bool hasScanned() const { return Scanned; }
-
-private:
- DependencyScanningService &Service;
- StringRef WorkingDirectory;
- DependencyConsumer &Consumer;
- DependencyActionController &Controller;
- IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
- std::optional<CompilerInstance> ScanInstanceStorage;
- std::shared_ptr<ModuleDepCollector> MDC;
- bool Scanned = false;
-};
-
// Helper functions and data types.
std::unique_ptr<DiagnosticOptions>
createDiagOptions(ArrayRef<std::string> CommandLine);
diff --git a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
index 2a264269652ca..c636da996cb71 100644
--- a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
+++ b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
@@ -18,14 +18,9 @@
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Option/Option.h"
-#include "llvm/Support/AdvisoryLock.h"
-#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Host.h"
-#include <mutex>
-#include <thread>
-
using namespace clang;
using namespace dependencies;
@@ -528,291 +523,3 @@ dependencies::initializeScanInstanceDependencyCollector(
ScanInstance.addDependencyCollector(MDC);
return MDC;
}
-
-/// Manages (and terminates) the asynchronous compilation of modules.
-class AsyncModuleCompiles {
- std::mutex Mutex;
- bool Stop = false;
- // FIXME: Have the service own a thread pool and use that instead.
- std::vector<std::thread> Compiles;
-
-public:
- /// Registers the module compilation, unless this instance is about to be
- /// destroyed.
- void add(llvm::unique_function<void()> Compile) {
- std::lock_guard<std::mutex> Lock(Mutex);
- if (!Stop)
- Compiles.emplace_back(std::move(Compile));
- }
-
- ~AsyncModuleCompiles() {
- {
- // Prevent registration of further module compiles.
- std::lock_guard<std::mutex> Lock(Mutex);
- Stop = true;
- }
-
- // Wait for outstanding module compiles to finish.
- for (std::thread &Compile : Compiles)
- Compile.join();
- }
-};
-
-struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
- DependencyScanningService &Service;
- DependencyActionController &Controller;
- AsyncModuleCompiles &Compiles;
-
- SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
- DependencyActionController &Controller,
- AsyncModuleCompiles &Compiles)
- : Service(Service), Controller(Controller), Compiles(Compiles) {}
-
- bool BeginSourceFileAction(CompilerInstance &CI) override;
-};
-
-/// The preprocessor callback that takes care of initiating an asynchronous
-/// module compilation if needed.
-struct AsyncModuleCompile : PPCallbacks {
- CompilerInstance &CI;
- DependencyScanningService &Service;
- DependencyActionController &Controller;
- AsyncModuleCompiles &Compiles;
-
- AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
- DependencyActionController &Controller,
- AsyncModuleCompiles &Compiles)
- : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
-
- void moduleLoadSkipped(Module *M) override {
- M = M->getTopLevelModule();
-
- HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
- ModuleCache &ModCache = CI.getModuleCache();
- ModuleFileName ModuleFileName = HS.getCachedModuleFileName(M);
-
- uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName);
- // Someone else already built/validated the PCM.
- if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
- return;
-
- if (!CI.getASTReader())
- CI.createASTReader();
- SmallVector<ASTReader::ImportedModule, 0> Imported;
- // Only calling ReadASTCore() to avoid the expensive eager deserialization
- // of the clang::Module objects in ReadAST().
- // FIXME: Consider doing this in the new thread depending on how expensive
- // the read turns out to be.
- switch (CI.getASTReader()->ReadASTCore(
- ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(),
- nullptr, Imported, {}, {}, {},
- ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
- ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
- case ASTReader::Success:
- // We successfully read a valid, up-to-date PCM.
- // FIXME: This could update the timestamp. Regular calls to
- // ASTReader::ReadAST() would do so unless they encountered corrupted
- // AST block, corrupted extension block, or did not read the expected
- // top-level module.
- return;
- case ASTReader::OutOfDate:
- case ASTReader::Missing:
- // The most interesting case.
- break;
- default:
- // Let the regular scan diagnose this.
- return;
- }
-
- auto Lock = ModCache.getLock(ModuleFileName);
- bool Owned;
- llvm::Error LockErr = Lock->tryLock().moveInto(Owned);
- // Someone else is building the PCM right now.
- if (!LockErr && !Owned)
- return;
- // We should build the PCM.
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
- llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
- Service, Service.getOpts().MakeVFS());
- VFS = createVFSFromCompilerInvocation(CI.getInvocation(),
- CI.getDiagnostics(), std::move(VFS));
- auto DC = std::make_unique<DiagnosticConsumer>();
- auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries(),
- Service.getLogger());
- CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
- std::move(MC));
- auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
- CloneConfig);
- auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
- CloneConfig);
-
- auto ModController = Controller.clone();
-
- // Note: This lock belongs to a module cache that might not outlive the
- // thread. This works, because the in-process lock only refers to an object
- // managed by the service, which does outlive the thread.
- Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
- ModCI2 = std::move(ModCI2), DC = std::move(DC),
- ModController = std::move(ModController), Service = &Service,
- Compiles = &Compiles] {
- llvm::CrashRecoveryContext CRC;
- (void)CRC.RunSafely([&] {
- // Quickly discovers and compiles modules for the real scan below.
- SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
- *Compiles);
- (void)ModCI1->ExecuteAction(Action1);
- // The real scan below.
- ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
- GenerateModuleFromModuleMapAction Action2;
- (void)ModCI2->ExecuteAction(Action2);
- });
- });
- }
-};
-
-/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
-/// modules asynchronously without blocking or importing them.
-struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
- DependencyScanningService &Service;
- DependencyActionController &Controller;
- AsyncModuleCompiles &Compiles;
-
- SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
- DependencyActionController &Controller,
- AsyncModuleCompiles &Compiles)
- : Service(Service), Controller(Controller), Compiles(Compiles) {}
-
- bool BeginSourceFileAction(CompilerInstance &CI) override {
- CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
- CI.getPreprocessor().addPPCallbacks(std::make_unique<AsyncModuleCompile>(
- CI, Service, Controller, Compiles));
- return true;
- }
-};
-
-bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
- CompilerInstance &CI) {
- CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
- CI.getPreprocessor().addPPCallbacks(
- std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles));
- return true;
-}
-
-bool DependencyScanningAction::runInvocation(
- std::string Executable,
- std::unique_ptr<CompilerInvocation> OriginalInvocation,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- DiagnosticConsumer *DiagConsumer) {
- // Making sure that we canonicalize the defines early to avoid unnecessary
- // variants in both the scanner and in the resulting explicit command lines.
- if (any(Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
- canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
-
- if (Scanned) {
- CompilerInstance &ScanInstance = *ScanInstanceStorage;
-
- // Scanning runs once for the first -cc1 invocation in a chain of driver
- // jobs. For any dependent jobs, reuse the scanning result and just
- // update the new invocation.
- // FIXME: to support multi-arch builds, each arch requires a separate scan
- if (MDC)
- MDC->applyDiscoveredDependencies(*OriginalInvocation);
-
- bool Success = OriginalInvocation->withCowRef<bool>(
- [&](CowCompilerInvocation &CowOriginalInvocation) {
- return Controller.finalize(ScanInstance, CowOriginalInvocation);
- });
- if (!Success)
- return false;
-
- Consumer.handleBuildCommand(
- {Executable, OriginalInvocation->getCC1CommandLine()});
- return true;
- }
-
- Scanned = true;
-
- // Create a compiler instance to handle the actual work.
- auto ScanInvocation =
- createScanCompilerInvocation(*OriginalInvocation, Service, Controller);
-
- // Quickly discovers and compiles modules for the real scan below.
- std::optional<AsyncModuleCompiles> AsyncCompiles;
- if (Service.getOpts().AsyncScanModules) {
- auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries(),
- Service.getLogger());
- auto ScanInstanceStorage = std::make_unique<CompilerInstance>(
- std::make_shared<CompilerInvocation>(*ScanInvocation), PCHContainerOps,
- std::move(ModCache));
- CompilerInstance &ScanInstance = *ScanInstanceStorage;
-
- DiagnosticConsumer DiagConsumer;
- initializeScanCompilerInstance(ScanInstance, FS, &DiagConsumer, Service,
- DepFS);
-
- // FIXME: Do this only once.
- SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
- auto MaybePrebuiltModulesASTMap =
- computePrebuiltModulesASTMap(ScanInstance, StableDirs);
- if (!MaybePrebuiltModulesASTMap)
- return false;
-
- // Normally this would be handled by GeneratePCHAction
- if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
- ScanInstance.getLangOpts().CompilingPCH = true;
-
- AsyncCompiles.emplace();
- SingleTUWithAsyncModuleCompiles Action(Service, Controller, *AsyncCompiles);
- (void)ScanInstance.ExecuteAction(Action);
- }
-
- auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries(),
- Service.getLogger());
- ScanInstanceStorage.emplace(std::move(ScanInvocation),
- std::move(PCHContainerOps), std::move(ModCache));
- CompilerInstance &ScanInstance = *ScanInstanceStorage;
-
- initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
- DepFS);
-
- llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
- auto MaybePrebuiltModulesASTMap =
- computePrebuiltModulesASTMap(ScanInstance, StableDirs);
- if (!MaybePrebuiltModulesASTMap)
- return false;
-
- auto DepOutputOpts = createDependencyOutputOptions(*OriginalInvocation);
-
- MDC = initializeScanInstanceDependencyCollector(
- ScanInstance, std::move(DepOutputOpts), Service, *OriginalInvocation,
- Controller, *MaybePrebuiltModulesASTMap, StableDirs);
-
- if (ScanInstance.getDiagnostics().hasErrorOccurred())
- return false;
-
- if (!Controller.initialize(ScanInstance, *OriginalInvocation))
- return false;
-
- ReadPCHAndPreprocessAction Action;
- const bool Result = ScanInstance.ExecuteAction(Action);
-
- if (Result) {
- if (MDC) {
- MDC->run(Consumer);
- MDC->applyDiscoveredDependencies(*OriginalInvocation);
- }
-
- bool Success = OriginalInvocation->withCowRef<bool>(
- [&](CowCompilerInvocation &CowOriginalInvocation) {
- return Controller.finalize(ScanInstance, CowOriginalInvocation);
- });
- if (!Success)
- return false;
-
- Consumer.handleBuildCommand(
- {Executable, OriginalInvocation->getCC1CommandLine()});
- }
-
- return Result;
-}
diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
index 0921f8a96eaac..b7fb7de6a1040 100644
--- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
@@ -17,11 +17,194 @@
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/AdvisoryLock.h"
+#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/VirtualFileSystem.h"
+#include <mutex>
+#include <thread>
using namespace clang;
using namespace dependencies;
+namespace {
+/// Manages (and terminates) the asynchronous compilation of modules.
+class AsyncModuleCompiles {
+ std::mutex Mutex;
+ bool Stop = false;
+ // FIXME: Have the service own a thread pool and use that instead.
+ std::vector<std::thread> Compiles;
+
+public:
+ /// Registers the module compilation, unless this instance is about to be
+ /// destroyed.
+ void add(llvm::unique_function<void()> Compile) {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ if (!Stop)
+ Compiles.emplace_back(std::move(Compile));
+ }
+
+ ~AsyncModuleCompiles() {
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ Stop = true;
+ }
+ for (std::thread &Compile : Compiles)
+ Compile.join();
+ }
+};
+
+struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
+ DependencyScanningService &Service;
+ DependencyActionController &Controller;
+ AsyncModuleCompiles &Compiles;
+
+ SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles)
+ : Service(Service), Controller(Controller), Compiles(Compiles) {}
+
+ bool BeginSourceFileAction(CompilerInstance &CI) override;
+};
+
+/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
+/// modules asynchronously without blocking or importing them.
+struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
+ DependencyScanningService &Service;
+ DependencyActionController &Controller;
+ AsyncModuleCompiles &Compiles;
+
+ SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles)
+ : Service(Service), Controller(Controller), Compiles(Compiles) {}
+
+ bool BeginSourceFileAction(CompilerInstance &CI) override;
+};
+
+/// The preprocessor callback that takes care of initiating an asynchronous
+/// module compilation if needed.
+struct AsyncModuleCompile : PPCallbacks {
+ CompilerInstance &CI;
+ DependencyScanningService &Service;
+ DependencyActionController &Controller;
+ AsyncModuleCompiles &Compiles;
+
+ AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles)
+ : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
+
+ void moduleLoadSkipped(Module *M) override {
+ M = M->getTopLevelModule();
+
+ HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
+ ModuleCache &ModCache = CI.getModuleCache();
+ ModuleFileName ModuleFileName = HS.getCachedModuleFileName(M);
+
+ uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName);
+ // Someone else already built/validated the PCM.
+ if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
+ return;
+
+ if (!CI.getASTReader())
+ CI.createASTReader();
+ SmallVector<ASTReader::ImportedModule, 0> Imported;
+ // Only calling ReadASTCore() to avoid the expensive eager deserialization
+ // of the clang::Module objects in ReadAST().
+ // FIXME: Consider doing this in the new thread depending on how expensive
+ // the read turns out to be.
+ switch (CI.getASTReader()->ReadASTCore(
+ ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(),
+ nullptr, Imported, {}, {}, {},
+ ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
+ ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
+ case ASTReader::Success:
+ // We successfully read a valid, up-to-date PCM.
+ // FIXME: This could update the timestamp. Regular calls to
+ // ASTReader::ReadAST() would do so unless they encountered corrupted
+ // AST block, corrupted extension block, or did not read the expected
+ // top-level module.
+ return;
+ case ASTReader::OutOfDate:
+ case ASTReader::Missing:
+ // The most interesting case.
+ break;
+ default:
+ // Let the regular scan diagnose this.
+ return;
+ }
+
+ auto Lock = ModCache.getLock(ModuleFileName);
+ bool Owned;
+ llvm::Error LockErr = Lock->tryLock().moveInto(Owned);
+ // Someone else is building the PCM right now.
+ if (!LockErr && !Owned)
+ return;
+ // We should build the PCM.
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
+ llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
+ Service, Service.getOpts().MakeVFS());
+ VFS = createVFSFromCompilerInvocation(CI.getInvocation(),
+ CI.getDiagnostics(), std::move(VFS));
+ auto DC = std::make_unique<DiagnosticConsumer>();
+ auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries(),
+ Service.getLogger());
+ CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
+ std::move(MC));
+ auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
+ CloneConfig);
+ auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
+ CloneConfig);
+
+ auto ModController = Controller.clone();
+
+ // Note: This lock belongs to a module cache that might not outlive the
+ // thread. This works, because the in-process lock only refers to an object
+ // managed by the service, which does outlive the thread.
+ Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
+ ModCI2 = std::move(ModCI2), DC = std::move(DC),
+ ModController = std::move(ModController), Service = &Service,
+ Compiles = &Compiles] {
+ llvm::CrashRecoveryContext CRC;
+ (void)CRC.RunSafely([&] {
+ // Quickly discovers and compiles modules for the real scan below.
+ SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
+ *Compiles);
+ (void)ModCI1->ExecuteAction(Action1);
+ // The real scan below.
+ ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
+ GenerateModuleFromModuleMapAction Action2;
+ (void)ModCI2->ExecuteAction(Action2);
+ });
+ });
+ }
+};
+
+bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
+ CompilerInstance &CI) {
+ CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
+ CI.getPreprocessor().addPPCallbacks(
+ std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles));
+ return true;
+}
+
+bool SingleTUWithAsyncModuleCompiles::BeginSourceFileAction(
+ CompilerInstance &CI) {
+ CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
+ CI.getPreprocessor().addPPCallbacks(
+ std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles));
+ return true;
+}
+} // namespace
+
+static void runTUModulePrescan(CompilerInstance &PrescanCI,
+ DependencyScanningService &Service,
+ DependencyActionController &Controller,
+ AsyncModuleCompiles &Compiles) {
+ SingleTUWithAsyncModuleCompiles Action(Service, Controller, Compiles);
+ (void)PrescanCI.ExecuteAction(Action);
+}
+
namespace clang {
namespace dependencies {
class CompilerInstanceWithContext {
@@ -30,10 +213,6 @@ class CompilerInstanceWithContext {
llvm::StringRef CWD;
std::vector<std::string> CommandLine;
- // Context - Diagnostics engine.
- DiagnosticConsumer *DiagConsumer = nullptr;
- std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts;
-
// Context - compiler invocation
std::unique_ptr<CompilerInvocation> OriginalInvocation;
@@ -44,6 +223,9 @@ class CompilerInstanceWithContext {
llvm::SmallVector<StringRef> StableDirs;
PrebuiltModulesAttrsMap PrebuiltModuleASTMap;
+ // Context - used by AsyncScan's prescan pass
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS;
+
// Compiler Instance
std::unique_ptr<CompilerInstance> CIPtr;
@@ -64,21 +246,26 @@ class CompilerInstanceWithContext {
CommandLine);
}
assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
- DiagEngineWithCmdAndOpts = std::move(DiagEngineWithDiagOpts);
- DiagConsumer = DiagEngineWithCmdAndOpts->DiagEngine->getClient();
-
- assert(OverlayFS && "OverlayFS required!");
- auto FS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
+ ScanFS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS));
OriginalInvocation = createCompilerInvocation(
- CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine);
+ CommandLine, *DiagEngineWithDiagOpts->DiagEngine);
if (!OriginalInvocation) {
- DiagEngineWithCmdAndOpts->DiagEngine->Report(
+ DiagEngineWithDiagOpts->DiagEngine->Report(
diag::err_fe_expected_compiler_job)
<< llvm::join(CommandLine, " ");
return false;
}
+ return initializeScanInstance(
+ Controller, DiagEngineWithDiagOpts->DiagEngine->getClient());
+ }
+
+ bool initializeScanInstance(DependencyActionController &Controller,
+ DiagnosticConsumer *DiagConsumer) {
+ assert(OriginalInvocation && ScanFS &&
+ "OriginalInvocation and ScanFS must be set before this call");
+
if (any(Worker.Service.getOpts().OptimizeArgs &
ScanningOptimizations::Macros))
canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
@@ -92,9 +279,8 @@ class CompilerInstanceWithContext {
Worker.PCHContainerOps, std::move(ModCache));
auto &CI = *CIPtr;
- initializeScanCompilerInstance(
- CI, std::move(FS), DiagEngineWithCmdAndOpts->DiagEngine->getClient(),
- Worker.Service, Worker.DepFS);
+ initializeScanCompilerInstance(CI, ScanFS, DiagConsumer, Worker.Service,
+ Worker.DepFS);
StableDirs = getInitialStableDirs(CI);
auto MaybePrebuiltModulesASTMap =
@@ -113,6 +299,31 @@ class CompilerInstanceWithContext {
return CI.createTarget();
}
+ bool prescanModulesAsync(AsyncModuleCompiles &Compiles,
+ DependencyActionController &Controller) {
+ auto ModCache = makeInProcessModuleCache(
+ Worker.Service.getModuleCacheEntries(), Worker.Service.getLogger());
+ CompilerInstance PrescanCI(
+ std::make_shared<CompilerInvocation>(CIPtr->getInvocation()),
+ Worker.PCHContainerOps, std::move(ModCache));
+
+ DiagnosticConsumer DiagConsumer;
+ initializeScanCompilerInstance(PrescanCI, ScanFS, &DiagConsumer,
+ Worker.Service, Worker.DepFS);
+
+ // FIXME: reuse the StableDirs/PrebuiltModuleASTMap computed in
+ // initialize().
+ SmallVector<StringRef> PrescanStableDirs = getInitialStableDirs(PrescanCI);
+ if (!computePrebuiltModulesASTMap(PrescanCI, PrescanStableDirs))
+ return false;
+
+ if (PrescanCI.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
+ PrescanCI.getLangOpts().CompilingPCH = true;
+
+ runTUModulePrescan(PrescanCI, Worker.Service, Controller, Compiles);
+ return true;
+ }
+
public:
static std::optional<CompilerInstanceWithContext>
initializeFromCC1Commandline(
@@ -239,6 +450,58 @@ class CompilerInstanceWithContext {
return true;
}
+
+ std::shared_ptr<ModuleDepCollector>
+ scanTranslationUnit(DependencyConsumer &Consumer,
+ DependencyActionController &Controller) {
+ assert(CIPtr && "CIPtr must be initialized before calling this method");
+ auto &CI = *CIPtr;
+
+ std::optional<AsyncModuleCompiles> AsyncCompiles;
+ if (Worker.Service.getOpts().AsyncScanModules) {
+ AsyncCompiles.emplace();
+ if (!prescanModulesAsync(*AsyncCompiles, Controller))
+ return nullptr;
+ }
+
+ auto MDC = initializeScanInstanceDependencyCollector(
+ CI, std::make_unique<DependencyOutputOptions>(*OutputOpts),
+ Worker.Service, *OriginalInvocation, Controller, PrebuiltModuleASTMap,
+ StableDirs);
+
+ if (CI.getDiagnostics().hasErrorOccurred())
+ return nullptr;
+
+ if (!Controller.initialize(CI, *OriginalInvocation))
+ return nullptr;
+
+ ReadPCHAndPreprocessAction Action;
+ if (!CI.ExecuteAction(Action))
+ return nullptr;
+
+ MDC->run(Consumer);
+ if (!applyAndReport(*MDC, *OriginalInvocation, Consumer, Controller,
+ CommandLine[0]))
+ return nullptr;
+ return MDC;
+ }
+
+ bool applyAndReport(ModuleDepCollector &MDC,
+ CompilerInvocation &ModuleInvocation,
+ DependencyConsumer &Consumer,
+ DependencyActionController &Controller,
+ StringRef Executable) {
+ MDC.applyDiscoveredDependencies(ModuleInvocation);
+ bool Success = ModuleInvocation.withCowRef<bool>(
+ [&](CowCompilerInvocation &CowModuleInvocation) {
+ return Controller.finalize(*CIPtr, CowModuleInvocation);
+ });
+ if (!Success)
+ return false;
+ Consumer.handleBuildCommand(
+ {Executable.str(), ModuleInvocation.getCC1CommandLine()});
+ return true;
+ }
};
} // namespace dependencies
} // namespace clang
@@ -267,20 +530,6 @@ DependencyScanningWorker::DependencyScanningWorker(
DependencyScanningWorker::~DependencyScanningWorker() = default;
-static bool createAndRunToolInvocation(
- ArrayRef<std::string> CommandLine, DependencyScanningAction &Action,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- std::shared_ptr<clang::PCHContainerOperations> &PCHContainerOps,
- DiagnosticsEngine &Diags) {
- auto Invocation = createCompilerInvocation(CommandLine, Diags);
- if (!Invocation)
- return false;
-
- return Action.runInvocation(CommandLine[0], std::move(Invocation),
- std::move(FS), PCHContainerOps,
- Diags.getClient());
-}
-
IntrusiveRefCntPtr<llvm::vfs::FileSystem>
DependencyScanningWorker::makeEffectiveVFS(
StringRef WorkingDirectory,
@@ -301,10 +550,11 @@ bool DependencyScanningWorker::computeDependencies(
DependencyConsumer &DepConsumer, DependencyActionController &Controller,
DiagnosticConsumer &DiagConsumer,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
- auto FS = makeEffectiveVFS(WorkingDirectory, std::move(OverlayFS));
+ auto FS = makeEffectiveVFS(WorkingDirectory, OverlayFS);
- DependencyScanningAction Action(Service, WorkingDirectory, DepConsumer,
- Controller, DepFS);
+ bool Scanned = false;
+ std::shared_ptr<ModuleDepCollector> MDC;
+ std::optional<CompilerInstanceWithContext> CIWC;
const bool Success = llvm::all_of(CommandLines, [&](const auto &Cmd) {
if (StringRef(Cmd[1]) != "-cc1") {
@@ -321,16 +571,43 @@ bool DependencyScanningWorker::computeDependencies(
});
auto DiagEngineWithDiagOpts =
- DiagnosticsEngineWithDiagOpts(Cmd, FS, DiagConsumer);
- auto &Diags = *DiagEngineWithDiagOpts.DiagEngine;
+ std::make_unique<DiagnosticsEngineWithDiagOpts>(Cmd, FS, DiagConsumer);
+ if (!Scanned) {
+ // Scanning runs once for the first -cc1 invocation in a chain of driver
+ // jobs.
+ // For any dependent jobs, reuse the scanning result and just update the
+ // new invocation.
+ // FIXME: to support multi-arch builds, each arch requires a separate
+ // scan.
+ Scanned = true;
+ auto Result = CompilerInstanceWithContext::initializeFromCC1Commandline(
+ *this, WorkingDirectory, Cmd, std::move(DiagEngineWithDiagOpts),
+ OverlayFS, Controller);
+ if (!Result)
+ return false;
+ CIWC.emplace(std::move(*Result));
+ MDC = CIWC->scanTranslationUnit(DepConsumer, Controller);
+ return MDC != nullptr;
+ }
+
+ auto Invocation =
+ createCompilerInvocation(Cmd, *DiagEngineWithDiagOpts->DiagEngine);
+ if (!Invocation)
+ return false;
+
+ // The first cc1 is canonicalized in initializeScanInstance; each sibling
+ // invocation must likewise be canonicalized before its cc1 command line is
+ // emitted. This is mostly relevant for multi-arch jobs where we currently
+ // do not do re-scans.
+ if (any(Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
+ canonicalizeDefines(Invocation->getPreprocessorOpts());
- // Create an invocation that uses the underlying file system to ensure that
- // any file system requests that are made by the driver do not go through
- // the dependency scanning filesystem.
- return createAndRunToolInvocation(Cmd, Action, FS, PCHContainerOps, Diags);
+ assert(CIWC && "Must have an initialized CIWC");
+ return CIWC->applyAndReport(*MDC, *Invocation, DepConsumer, Controller,
+ Cmd.front());
});
- return Success && Action.hasScanned();
+ return Success && Scanned;
}
bool DependencyScanningWorker::computeDependenciesByName(
diff --git a/clang/test/ClangScanDeps/logging-simple.c b/clang/test/ClangScanDeps/logging-simple.c
index d85d660317c75..0022fb97cbc73 100644
--- a/clang/test/ClangScanDeps/logging-simple.c
+++ b/clang/test/ClangScanDeps/logging-simple.c
@@ -14,6 +14,7 @@
// less, strictly in this order. Changes to this list should be intentional.
// CHECK: [{{[0-9]+\.[0-9]+}}] [[#PID:]] [[#TID:]]: starting scanning command:{{.*}}tu.c
+// CHECK-NEXT: [{{[0-9]+\.[0-9]+}}] [[#PID]] [[#TID]]: init_compiler_instance_with_context:{{.*}}
// CHECK-NEXT: [{{[0-9]+\.[0-9]+}}] [[#PID]] [[#TID]]: timestamp_read: {{.*}}[[PCMFILE:.*\.pcm]]
// CHECK-NEXT: [{{[0-9]+\.[0-9]+}}] [[#PID]] [[#TID]]: pcm_read_cached: {{.*}}[[PCMFILE]]
// CHECK-NEXT: [{{[0-9]+\.[0-9]+}}] [[#PID]] [[#TID]]: pcm_read_disk: {{.*}}[[PCMFILE]]
diff --git a/clang/unittests/Tooling/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScannerTest.cpp
index 6b119000000cf..62782f8a56945 100644
--- a/clang/unittests/Tooling/DependencyScannerTest.cpp
+++ b/clang/unittests/Tooling/DependencyScannerTest.cpp
@@ -304,3 +304,38 @@ TEST(DependencyScanner, ScanDepsWithModuleLookup) {
EXPECT_TRUE(!llvm::is_contained(InterceptFS->StatPaths, OtherPath));
EXPECT_EQ(InterceptFS->ReadFiles, std::vector<std::string>{"test.m"});
}
+
+// When scanning from a TU buffer, the in-memory TU lives ONLY
+// in the overlay filesystem built from that buffer, never on the base VFS. If
+// DependencyScanningWorker::computeDependencies moves the overlay away before
+// initializing the scan CompilerInstance, the scanner falls back to the base
+// VFS, cannot find its input, and the scan fails.
+TEST(DependencyScanner, ScanDepsTUBufferOverlayReachesScan) {
+ std::vector<std::string> CommandLine = {
+ "clang", "-target", "x86_64-apple-macosx10.7", "-c", "-o", "tu.o"};
+ StringRef CWD = "/root";
+
+ // Base VFS intentionally does NOT contain the TU file.
+ auto VFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+ VFS->setCurrentWorkingDirectory(CWD);
+
+ DependencyScanningServiceOptions Opts;
+ Opts.MakeVFS = [&] { return VFS; };
+ DependencyScanningService Service(std::move(Opts));
+ DependencyScanningTool ScanTool(Service);
+
+ auto Sept = llvm::sys::path::get_separator();
+ std::string TUPath = std::string(llvm::formatv("{0}root{0}tu.c", Sept));
+ auto TU = llvm::MemoryBuffer::getMemBuffer("int main(void) { return 0; }\n",
+ TUPath);
+
+ TextDiagnosticBuffer DiagConsumer;
+ llvm::DenseSet<ModuleID> AlreadySeen;
+ auto Result = ScanTool.getTranslationUnitDependencies(
+ CommandLine, CWD, DiagConsumer, AlreadySeen,
+ CallbackActionController::lookupUnreachableModuleOutput,
+ TU->getMemBufferRef());
+ ASSERT_TRUE(Result.has_value());
+ EXPECT_TRUE(llvm::any_of(Result->FileDeps,
+ [](StringRef F) { return F.contains("tu.c"); }));
+}
More information about the cfe-commits
mailing list