[clang] [clang][Dependency Scanning] Refactor Scanning Compiler Instance Initialization (PR #161300)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Sep 29 16:56:30 PDT 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang
Author: Qiongsi Wu (qiongsiwu)
<details>
<summary>Changes</summary>
This PR follows https://github.com/llvm/llvm-project/pull/160795, and it is the second of a series of planned PRs to land https://github.com/llvm/llvm-project/pull/160207 in smaller pieces.
The initialization steps within `DependencyScanningAction::runInvocation` are broken up in to several helper functions. The intention is to reuse the helper functions in a followup PR to initialize the `CompilerInstanceWithContext`.
This PR is an NFC in spirit but it changes some execution orders in `DependencyScanningAction::runInvocation`. The code to initialize the scanner instance's option flags are collected together in `initializeScanCompilerInstance`, in contrast to the current way where the flags are set in piece-meal fashion before and after we compute the `StableDirs` and the `rebuiltModulesASTMap`.
Part of work for rdar://136303612.
---
Patch is 25.08 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/161300.diff
3 Files Affected:
- (modified) clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp (+183-43)
- (modified) clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h (+52-1)
- (modified) clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp (+19-84)
``````````diff
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp
index d370bfd0dd10f..5a16edf859193 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp
@@ -9,8 +9,10 @@
#include "DependencyScannerImpl.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/DiagnosticSerialization.h"
+#include "clang/Driver/Driver.h"
#include "clang/Frontend/FrontendActions.h"
#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
+#include "llvm/TargetParser/Host.h"
using namespace clang;
using namespace tooling;
@@ -334,6 +336,17 @@ class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
};
} // namespace
+std::unique_ptr<DiagnosticOptions>
+clang::tooling::dependencies::createDiagOptions(
+ const std::vector<std::string> &CommandLine) {
+ std::vector<const char *> CLI;
+ for (const std::string &Arg : CommandLine)
+ CLI.push_back(Arg.c_str());
+ auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
+ sanitizeDiagOpts(*DiagOpts);
+ return DiagOpts;
+}
+
/// Sanitize diagnostic options for dependency scan.
void clang::tooling::dependencies::sanitizeDiagOpts(
DiagnosticOptions &DiagOpts) {
@@ -356,43 +369,98 @@ void clang::tooling::dependencies::sanitizeDiagOpts(
});
}
-bool DependencyScanningAction::runInvocation(
- std::shared_ptr<CompilerInvocation> Invocation,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
- std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- DiagnosticConsumer *DiagConsumer) {
- // Making sure that we canonicalize the defines before we create the deep
- // copy to avoid unnecessary variants in the scanner and in the resulting
- // explicit command lines.
- if (any(Service.getOptimizeArgs() & ScanningOptimizations::Macros))
- canonicalizeDefines(Invocation->getPreprocessorOpts());
+std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>>
+clang::tooling::dependencies::buildCompilation(
+ ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
+ SmallVector<const char *, 256> Argv;
+ Argv.reserve(ArgStrs.size());
+ for (const std::string &Arg : ArgStrs)
+ Argv.push_back(Arg.c_str());
+
+ std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
+ Argv[0], llvm::sys::getDefaultTargetTriple(), Diags,
+ "clang LLVM compiler", FS);
+ Driver->setTitle("clang_based_tool");
+
+ llvm::BumpPtrAllocator Alloc;
+ bool CLMode = driver::IsClangCL(
+ driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1)));
+
+ if (llvm::Error E =
+ driver::expandResponseFiles(Argv, CLMode, Alloc, FS.get())) {
+ Diags.Report(diag::err_drv_expand_response_file)
+ << llvm::toString(std::move(E));
+ return std::make_pair(nullptr, nullptr);
+ }
- // Make a deep copy of the original Clang invocation.
- CompilerInvocation OriginalInvocation(*Invocation);
+ std::unique_ptr<driver::Compilation> Compilation(
+ Driver->BuildCompilation(llvm::ArrayRef(Argv)));
+ if (!Compilation)
+ return std::make_pair(nullptr, nullptr);
- if (Scanned) {
- // Scanning runs once for the first -cc1 invocation in a chain of driver
- // jobs. For any dependent jobs, reuse the scanning result and just
- // update the LastCC1Arguments to correspond to the new invocation.
- // FIXME: to support multi-arch builds, each arch requires a separate scan
- setLastCC1Arguments(std::move(OriginalInvocation));
- return true;
- }
+ if (Compilation->containsError())
+ return std::make_pair(nullptr, nullptr);
+ ;
- Scanned = true;
+ return std::make_pair(std::move(Driver), std::move(Compilation));
+}
- // Create a compiler instance to handle the actual work.
- auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
- ScanInstanceStorage.emplace(std::move(Invocation), std::move(PCHContainerOps),
- ModCache.get());
- CompilerInstance &ScanInstance = *ScanInstanceStorage;
+llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+clang::tooling::dependencies::initVFSForByNameScanning(
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
+ std::vector<std::string> &CommandLine, StringRef WorkingDirectory,
+ StringRef FakeFileNamePrefix) {
+ // Reset what might have been modified in the previous worker invocation.
+ BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
+
+ // If we're scanning based on a module name alone, we don't expect the client
+ // to provide us with an input file. However, the driver really wants to have
+ // one. Let's just make it up to make the driver happy.
+ auto OverlayFS =
+ llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
+ auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+ InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
+ SmallString<128> FakeInputPath;
+ // TODO: We should retry the creation if the path already exists.
+ llvm::sys::fs::createUniquePath(FakeFileNamePrefix + "-%%%%%%%%.input",
+ FakeInputPath,
+ /*MakeAbsolute=*/false);
+ InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
+ OverlayFS->pushOverlay(InMemoryOverlay);
+
+ CommandLine.emplace_back(FakeInputPath);
+
+ return OverlayFS;
+}
+
+std::unique_ptr<CompilerInvocation>
+clang::tooling::dependencies::createCompilerInvocation(
+ const std::vector<std::string> CommandLine, DiagnosticsEngine &Diags) {
+ llvm::opt::ArgStringList Argv;
+ for (const std::string &Str : ArrayRef(CommandLine).drop_front())
+ Argv.push_back(Str.c_str());
+
+ auto Invocation = std::make_unique<CompilerInvocation>();
+ if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) {
+ // FIXME: Should we just go on like cc1_main does?
+ return nullptr;
+ }
+ return Invocation;
+}
+
+bool clang::tooling::dependencies::initializeScanCompilerInstance(
+ CompilerInstance &ScanInstance,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+ DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
+ llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
ScanInstance.setBuildingModule(false);
ScanInstance.createVirtualFileSystem(FS, DiagConsumer);
// Create the compiler's actual diagnostics engine.
sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
- assert(!DiagConsumerFinished && "attempt to reuse finished consumer");
ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
if (!ScanInstance.hasDiagnostics())
return false;
@@ -434,6 +502,26 @@ bool DependencyScanningAction::runInvocation(
ScanInstance.createSourceManager(*FileMgr);
+ // Consider different header search and diagnostic options to create
+ // different modules. This avoids the unsound aliasing of module PCMs.
+ //
+ // TODO: Implement diagnostic bucketing to reduce the impact of strict
+ // context hashing.
+ ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
+ ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
+ ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
+ ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
+ ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = true;
+ ScanInstance.getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
+
+ // Avoid some checks and module map parsing when loading PCM files.
+ ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false;
+
+ return true;
+}
+
+llvm::SmallVector<StringRef> clang::tooling::dependencies::computeStableDirs(
+ CompilerInstance &ScanInstance) {
// Create a collection of stable directories derived from the ScanInstance
// for determining whether module dependencies would fully resolve from
// those directories.
@@ -441,7 +529,12 @@ bool DependencyScanningAction::runInvocation(
const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot))
StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
+ return StableDirs;
+}
+std::optional<PrebuiltModulesAttrsMap>
+clang::tooling::dependencies::computePrebuiltModulesASTMap(
+ CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {
// Store a mapping of prebuilt module files and their properties like header
// search options. This will prevent the implicit build to create duplicate
// modules and will force reuse of the existing prebuilt module files
@@ -453,8 +546,18 @@ bool DependencyScanningAction::runInvocation(
ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs))
- return false;
+ return {};
+
+ return PrebuiltModulesASTMap;
+}
+void clang::tooling::dependencies::initializeModuleDepCollector(
+ CompilerInstance &ScanInstance, std::shared_ptr<ModuleDepCollector> &MDC,
+ StringRef WorkingDirectory, DependencyConsumer &Consumer,
+ DependencyScanningService &Service, CompilerInvocation &Inv,
+ DependencyActionController &Controller,
+ PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
+ llvm::SmallVector<StringRef> &StableDirs) {
// Create the dependency collector that will collect the produced
// dependencies.
//
@@ -463,6 +566,11 @@ bool DependencyScanningAction::runInvocation(
// which ensures that the compiler won't create new dependency collectors,
// and thus won't write out the extra '.d' files to disk.
auto Opts = std::make_unique<DependencyOutputOptions>();
+
+ // We rely on the behaviour that that ScanInstance's Invocation instance's
+ // dependency output opts are cleared here.
+ // TODO: we will need to preserve and recover the original dependency output
+ // opts if we want to reuse ScanInstance.
std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
// We need at least one -MT equivalent for the generator of make dependency
// files to work.
@@ -480,26 +588,58 @@ bool DependencyScanningAction::runInvocation(
case ScanningOutputFormat::P1689:
case ScanningOutputFormat::Full:
MDC = std::make_shared<ModuleDepCollector>(
- Service, std::move(Opts), ScanInstance, Consumer, Controller,
- OriginalInvocation, std::move(PrebuiltModulesASTMap), StableDirs);
+ Service, std::move(Opts), ScanInstance, Consumer, Controller, Inv,
+ std::move(PrebuiltModulesASTMap), StableDirs);
ScanInstance.addDependencyCollector(MDC);
break;
}
+}
- // Consider different header search and diagnostic options to create
- // different modules. This avoids the unsound aliasing of module PCMs.
- //
- // TODO: Implement diagnostic bucketing to reduce the impact of strict
- // context hashing.
- ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
- ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
- ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
- ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
- ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = true;
- ScanInstance.getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
+bool DependencyScanningAction::runInvocation(
+ std::unique_ptr<CompilerInvocation> Invocation,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+ DiagnosticConsumer *DiagConsumer) {
+ // Making sure that we canonicalize the defines before we create the deep
+ // copy to avoid unnecessary variants in the scanner and in the resulting
+ // explicit command lines.
+ if (any(Service.getOptimizeArgs() & ScanningOptimizations::Macros))
+ canonicalizeDefines(Invocation->getPreprocessorOpts());
- // Avoid some checks and module map parsing when loading PCM files.
- ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false;
+ // Make a deep copy of the original Clang invocation.
+ CompilerInvocation OriginalInvocation(*Invocation);
+
+ if (Scanned) {
+ // Scanning runs once for the first -cc1 invocation in a chain of driver
+ // jobs. For any dependent jobs, reuse the scanning result and just
+ // update the LastCC1Arguments to correspond to the new invocation.
+ // FIXME: to support multi-arch builds, each arch requires a separate scan
+ setLastCC1Arguments(std::move(OriginalInvocation));
+ return true;
+ }
+
+ Scanned = true;
+
+ // Create a compiler instance to handle the actual work.
+ auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
+ ScanInstanceStorage.emplace(std::move(Invocation), std::move(PCHContainerOps),
+ ModCache.get());
+ CompilerInstance &ScanInstance = *ScanInstanceStorage;
+
+ assert(!DiagConsumerFinished && "attempt to reuse finished consumer");
+ if (!initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
+ DepFS))
+ return false;
+
+ llvm::SmallVector<StringRef> StableDirs = computeStableDirs(ScanInstance);
+ auto MaybePrebuiltModulesASTMap =
+ computePrebuiltModulesASTMap(ScanInstance, StableDirs);
+ if (!MaybePrebuiltModulesASTMap)
+ return false;
+
+ initializeModuleDepCollector(ScanInstance, MDC, WorkingDirectory, Consumer,
+ Service, OriginalInvocation, Controller,
+ *MaybePrebuiltModulesASTMap, StableDirs);
std::unique_ptr<FrontendAction> Action;
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h
index 32fbcfffde53c..4e01419b3af8e 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h
@@ -9,8 +9,10 @@
#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNER_H
#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNER_H
+#include "clang/Driver/Compilation.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
@@ -35,7 +37,7 @@ class DependencyScanningAction {
: Service(Service), WorkingDirectory(WorkingDirectory),
Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)),
ModuleName(ModuleName) {}
- bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
+ bool runInvocation(std::unique_ptr<CompilerInvocation> Invocation,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
DiagnosticConsumer *DiagConsumer);
@@ -73,8 +75,57 @@ class DependencyScanningAction {
};
// Helper functions
+std::unique_ptr<DiagnosticOptions>
+createDiagOptions(const std::vector<std::string> &CommandLine);
void sanitizeDiagOpts(DiagnosticOptions &DiagOpts);
+struct TextDiagnosticsPrinterWithOutput {
+ std::string DiagnosticOutput;
+ llvm::raw_string_ostream DiagnosticsOS;
+ std::unique_ptr<clang::DiagnosticOptions> DiagOpts;
+ TextDiagnosticPrinter DiagPrinter;
+
+ TextDiagnosticsPrinterWithOutput(const std::vector<std::string> &CommandLine)
+ : DiagnosticsOS(DiagnosticOutput),
+ DiagOpts(createDiagOptions(CommandLine)),
+ DiagPrinter(DiagnosticsOS, *DiagOpts) {}
+};
+
+std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>>
+buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
+
+std::unique_ptr<CompilerInvocation>
+createCompilerInvocation(const std::vector<std::string> CommandLine,
+ DiagnosticsEngine &Diags);
+
+llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+initVFSForByNameScanning(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
+ std::vector<std::string> &CommandLine,
+ StringRef WorkingDirectory,
+ StringRef FakeFileNamePrefix);
+
+bool initializeScanCompilerInstance(
+ CompilerInstance &ScanInstance,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+ DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
+ llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS);
+
+llvm::SmallVector<StringRef> computeStableDirs(CompilerInstance &ScanInstance);
+
+std::optional<PrebuiltModulesAttrsMap>
+computePrebuiltModulesASTMap(CompilerInstance &ScanInstance,
+ llvm::SmallVector<StringRef> &StableDirs);
+
+void initializeModuleDepCollector(CompilerInstance &ScanInstance,
+ std::shared_ptr<ModuleDepCollector> &MDC,
+ StringRef WorkingDirectory,
+ DependencyConsumer &Consumer,
+ DependencyScanningService &Service,
+ CompilerInvocation &Inv,
+ DependencyActionController &Controller,
+ PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
+ llvm::SmallVector<StringRef> &StableDirs);
} // namespace dependencies
} // namespace tooling
} // namespace clang
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 796e587ba9147..ddc5e9f26f4c3 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -63,32 +63,19 @@ DependencyScanningWorker::DependencyScanningWorker(
}
}
-static std::unique_ptr<DiagnosticOptions>
-createDiagOptions(const std::vector<std::string> &CommandLine) {
- std::vector<const char *> CLI;
- for (const std::string &Arg : CommandLine)
- CLI.push_back(Arg.c_str());
- auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
- sanitizeDiagOpts(*DiagOpts);
- return DiagOpts;
-}
-
llvm::Error DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
std::optional<llvm::MemoryBufferRef> TUBuffer) {
// Capture the emitted diagnostics and report them to the client
// in the case of a failure.
- std::string DiagnosticOutput;
- llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
- auto DiagOpts = createDiagOptions(CommandLine);
- TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, *DiagOpts);
+ TextDiagnosticsPrinterWithOutput DiagPrinterWithOS(CommandLine);
if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
- DiagPrinter, TUBuffer))
+ DiagPrinterWithOS.DiagPrinter, TUBuffer))
return llvm::Error::success();
- return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
- llvm::inconvertibleErrorCode());
+ return llvm::make_error<llvm::StringError>(
+ DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode());
}
llvm::Error DependencyScanningWorker::computeDependencies(
@@ -97,51 +84,24 @@ llvm::Error DependencyScanningWorker::computeDependencies(
StringRef ModuleName) {
// Capture the emitted diagnostics and report them to the client
// in the case of a failure.
- std::string DiagnosticOutput;
- llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
- auto DiagOpts = createDiagOptions(CommandLine);
- TextDiagnosticPrinter DiagPrinter(D...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/161300
More information about the cfe-commits
mailing list