[clang-tools-extra] [clangd] [C++20] [Modules] Introduce initial support for C++20 Modules (PR #66462)
Chuanqi Xu via cfe-commits
cfe-commits at lists.llvm.org
Sun Apr 14 22:56:45 PDT 2024
https://github.com/ChuanqiXu9 updated https://github.com/llvm/llvm-project/pull/66462
>From 32010ae7e0a47cd4a70a9401980b32ed1d3e10f6 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd at linux.alibaba.com>
Date: Fri, 15 Sep 2023 11:33:53 +0800
Subject: [PATCH] [clangd] [C++20] [Modules] Introduce initial support for
C++20 Modules
Alternatives to https://reviews.llvm.org/D153114.
Try to address https://github.com/clangd/clangd/issues/1293.
See the links for design ideas. We want to have some initial support in
clang18.
This is the initial support for C++20 Modules in clangd.
As suggested by sammccall in https://reviews.llvm.org/D153114,
we should minimize the scope of the initial patch to make it easier
to review and understand so that every one are in the same page:
> Don't attempt any cross-file or cross-version coordination: i.e. don't
> try to reuse BMIs between different files, don't try to reuse BMIs
> between (preamble) reparses of the same file, don't try to persist the
> module graph. Instead, when building a preamble, synchronously scan
> for the module graph, build the required PCMs on the single preamble
> thread with filenames private to that preamble, and then proceed to
> build the preamble.
And this patch reflects the above opinions.
---
clang-tools-extra/clangd/CMakeLists.txt | 4 +
clang-tools-extra/clangd/ClangdServer.cpp | 2 +
clang-tools-extra/clangd/ClangdServer.h | 3 +
.../clangd/GlobalCompilationDatabase.cpp | 23 ++
.../clangd/GlobalCompilationDatabase.h | 14 +
.../clangd/ModuleDependencyScanner.cpp | 81 +++++
.../clangd/ModuleDependencyScanner.h | 106 ++++++
clang-tools-extra/clangd/ModulesBuilder.cpp | 339 ++++++++++++++++++
clang-tools-extra/clangd/ModulesBuilder.h | 71 ++++
clang-tools-extra/clangd/ParsedAST.cpp | 7 +
clang-tools-extra/clangd/Preamble.cpp | 28 +-
clang-tools-extra/clangd/Preamble.h | 10 +
.../clangd/PrerequisiteModules.h | 87 +++++
clang-tools-extra/clangd/ProjectModules.cpp | 62 ++++
clang-tools-extra/clangd/ProjectModules.h | 55 +++
clang-tools-extra/clangd/TUScheduler.cpp | 50 ++-
clang-tools-extra/clangd/TUScheduler.h | 7 +
clang-tools-extra/clangd/test/CMakeLists.txt | 1 +
clang-tools-extra/clangd/test/modules.test | 83 +++++
clang-tools-extra/clangd/tool/Check.cpp | 13 +-
clang-tools-extra/clangd/tool/ClangdMain.cpp | 8 +
.../clangd/unittests/CMakeLists.txt | 2 +
.../clangd/unittests/CodeCompleteTests.cpp | 22 +-
.../clangd/unittests/FileIndexTests.cpp | 4 +-
.../unittests/ModuleDependencyScannerTest.cpp | 176 +++++++++
.../clangd/unittests/ModulesTestSetup.h | 103 ++++++
.../clangd/unittests/ParsedASTTests.cpp | 8 +-
.../clangd/unittests/PreambleTests.cpp | 6 +-
.../unittests/PrerequisiteModulesTest.cpp | 224 ++++++++++++
clang-tools-extra/clangd/unittests/TestTU.cpp | 14 +-
clang-tools-extra/docs/ReleaseNotes.rst | 3 +
31 files changed, 1576 insertions(+), 40 deletions(-)
create mode 100644 clang-tools-extra/clangd/ModuleDependencyScanner.cpp
create mode 100644 clang-tools-extra/clangd/ModuleDependencyScanner.h
create mode 100644 clang-tools-extra/clangd/ModulesBuilder.cpp
create mode 100644 clang-tools-extra/clangd/ModulesBuilder.h
create mode 100644 clang-tools-extra/clangd/PrerequisiteModules.h
create mode 100644 clang-tools-extra/clangd/ProjectModules.cpp
create mode 100644 clang-tools-extra/clangd/ProjectModules.h
create mode 100644 clang-tools-extra/clangd/test/modules.test
create mode 100644 clang-tools-extra/clangd/unittests/ModuleDependencyScannerTest.cpp
create mode 100644 clang-tools-extra/clangd/unittests/ModulesTestSetup.h
create mode 100644 clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp
diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index 3911fb6c6c746a..242a8ad2e350be 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -97,7 +97,10 @@ add_clang_library(clangDaemon
IncludeFixer.cpp
InlayHints.cpp
JSONTransport.cpp
+ ModuleDependencyScanner.cpp
+ ModulesBuilder.cpp
PathMapping.cpp
+ ProjectModules.cpp
Protocol.cpp
Quality.cpp
ParsedAST.cpp
@@ -161,6 +164,7 @@ clang_target_link_libraries(clangDaemon
clangAST
clangASTMatchers
clangBasic
+ clangDependencyScanning
clangDriver
clangFormat
clangFrontend
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index 13d788162817fb..8ba4b38c420ab6 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -202,6 +202,7 @@ ClangdServer::Options::operator TUScheduler::Options() const {
Opts.UpdateDebounce = UpdateDebounce;
Opts.ContextProvider = ContextProvider;
Opts.PreambleThrottler = PreambleThrottler;
+ Opts.ExperimentalModulesSupport = ExperimentalModulesSupport;
return Opts;
}
@@ -222,6 +223,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
DirtyFS(std::make_unique<DraftStoreFS>(TFS, DraftMgr)) {
if (Opts.AsyncThreadsCount != 0)
IndexTasks.emplace();
+
// Pass a callback into `WorkScheduler` to extract symbols from a newly
// parsed file and rebuild the file index synchronously each time an AST
// is parsed.
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index a416602251428b..dc546b118cb8f5 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -112,6 +112,9 @@ class ClangdServer {
/// This throttler controls which preambles may be built at a given time.
clangd::PreambleThrottler *PreambleThrottler = nullptr;
+ /// Enable experimental support for modules.
+ bool ExperimentalModulesSupport = false;
+
/// If true, ClangdServer builds a dynamic in-memory index for symbols in
/// opened files and uses the index to augment code completion results.
bool BuildDynamicSymbolIndex = false;
diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
index d1833759917a30..3a1931041eb0c5 100644
--- a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
+++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
@@ -9,6 +9,7 @@
#include "GlobalCompilationDatabase.h"
#include "Config.h"
#include "FS.h"
+#include "ProjectModules.h"
#include "SourceCode.h"
#include "support/Logger.h"
#include "support/Path.h"
@@ -729,6 +730,21 @@ DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const {
return Res->PI;
}
+std::shared_ptr<ProjectModules>
+DirectoryBasedGlobalCompilationDatabase::getProjectModules(PathRef File) const {
+ CDBLookupRequest Req;
+ Req.FileName = File;
+ Req.ShouldBroadcast = false;
+ Req.FreshTime = Req.FreshTimeMissing =
+ std::chrono::steady_clock::time_point::min();
+ auto Res = lookupCDB(Req);
+ if (!Res)
+ return {};
+ return ProjectModules::create(
+ ProjectModules::ProjectModulesKind::ScanningAllFiles,
+ Res->CDB->getAllFiles(), *this, Opts.TFS);
+}
+
OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base,
std::vector<std::string> FallbackFlags,
CommandMangler Mangler)
@@ -805,6 +821,13 @@ std::optional<ProjectInfo> DelegatingCDB::getProjectInfo(PathRef File) const {
return Base->getProjectInfo(File);
}
+std::shared_ptr<ProjectModules>
+DelegatingCDB::getProjectModules(PathRef File) const {
+ if (!Base)
+ return nullptr;
+ return Base->getProjectModules(File);
+}
+
tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const {
if (!Base)
return GlobalCompilationDatabase::getFallbackCommand(File);
diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.h b/clang-tools-extra/clangd/GlobalCompilationDatabase.h
index 2bf8c973c534c6..1bd6324c4cd8e4 100644
--- a/clang-tools-extra/clangd/GlobalCompilationDatabase.h
+++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.h
@@ -25,6 +25,8 @@
namespace clang {
namespace clangd {
+class ProjectModules;
+
struct ProjectInfo {
// The directory in which the compilation database was discovered.
// Empty if directory-based compilation database discovery was not used.
@@ -45,6 +47,12 @@ class GlobalCompilationDatabase {
return std::nullopt;
}
+ /// Get the modules in the closest project to \p File
+ virtual std::shared_ptr<ProjectModules>
+ getProjectModules(PathRef File) const {
+ return nullptr;
+ }
+
/// Makes a guess at how to build a file.
/// The default implementation just runs clang on the file.
/// Clangd should treat the results as unreliable.
@@ -76,6 +84,9 @@ class DelegatingCDB : public GlobalCompilationDatabase {
std::optional<ProjectInfo> getProjectInfo(PathRef File) const override;
+ std::shared_ptr<ProjectModules>
+ getProjectModules(PathRef File) const override;
+
tooling::CompileCommand getFallbackCommand(PathRef File) const override;
bool blockUntilIdle(Deadline D) const override;
@@ -122,6 +133,9 @@ class DirectoryBasedGlobalCompilationDatabase
/// \p File's parents.
std::optional<ProjectInfo> getProjectInfo(PathRef File) const override;
+ std::shared_ptr<ProjectModules>
+ getProjectModules(PathRef File) const override;
+
bool blockUntilIdle(Deadline Timeout) const override;
private:
diff --git a/clang-tools-extra/clangd/ModuleDependencyScanner.cpp b/clang-tools-extra/clangd/ModuleDependencyScanner.cpp
new file mode 100644
index 00000000000000..3413d0bb0eaa59
--- /dev/null
+++ b/clang-tools-extra/clangd/ModuleDependencyScanner.cpp
@@ -0,0 +1,81 @@
+//===---------------- ModuleDependencyScanner.cpp ----------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ModuleDependencyScanner.h"
+#include "support/Logger.h"
+
+namespace clang {
+namespace clangd {
+
+std::optional<ModuleDependencyScanner::ModuleDependencyInfo>
+ModuleDependencyScanner::scan(PathRef FilePath) {
+ std::optional<tooling::CompileCommand> Cmd = CDB.getCompileCommand(FilePath);
+
+ if (!Cmd)
+ return std::nullopt;
+
+ using namespace clang::tooling::dependencies;
+
+ llvm::SmallString<128> FilePathDir(FilePath);
+ llvm::sys::path::remove_filename(FilePathDir);
+ DependencyScanningTool ScanningTool(Service, TFS.view(FilePathDir));
+
+ llvm::Expected<P1689Rule> ScanningResult =
+ ScanningTool.getP1689ModuleDependencyFile(*Cmd, Cmd->Directory);
+
+ if (auto E = ScanningResult.takeError()) {
+ log("Scanning modules dependencies for {0} failed: {1}", FilePath,
+ llvm::toString(std::move(E)));
+ return std::nullopt;
+ }
+
+ ModuleDependencyInfo Result;
+
+ if (ScanningResult->Provides) {
+ ModuleNameToSource[ScanningResult->Provides->ModuleName] = FilePath;
+ Result.ModuleName = ScanningResult->Provides->ModuleName;
+ }
+
+ for (auto &Required : ScanningResult->Requires)
+ Result.RequiredModules.push_back(Required.ModuleName);
+
+ return Result;
+}
+
+void ModuleDependencyScanner::globalScan(
+ const std::vector<std::string> &AllFiles) {
+ for (auto &File : AllFiles)
+ scan(File);
+
+ GlobalScanned = true;
+}
+
+PathRef
+ModuleDependencyScanner::getSourceForModuleName(StringRef ModuleName) const {
+ assert(
+ GlobalScanned &&
+ "We should only call getSourceForModuleName after calling globalScan()");
+
+ if (auto It = ModuleNameToSource.find(ModuleName);
+ It != ModuleNameToSource.end())
+ return It->second;
+
+ return {};
+}
+
+std::vector<std::string>
+ModuleDependencyScanner::getRequiredModules(PathRef File) {
+ auto ScanningResult = scan(File);
+ if (!ScanningResult)
+ return {};
+
+ return ScanningResult->RequiredModules;
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/ModuleDependencyScanner.h b/clang-tools-extra/clangd/ModuleDependencyScanner.h
new file mode 100644
index 00000000000000..5251bdeadfb851
--- /dev/null
+++ b/clang-tools-extra/clangd/ModuleDependencyScanner.h
@@ -0,0 +1,106 @@
+//===-------------- ModuleDependencyScanner.h --------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULEDEPENDENCYSCANNER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULEDEPENDENCYSCANNER_H
+
+#include "GlobalCompilationDatabase.h"
+#include "support/Path.h"
+#include "support/ThreadsafeFS.h"
+
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringMap.h"
+
+namespace clang {
+namespace clangd {
+
+/// A scanner to query the dependency information for C++20 Modules.
+///
+/// The scanner can scan a single file with `scan(PathRef)` member function
+/// or scan the whole project with `globalScan(PathRef)` member function. See
+/// the comments of `globalScan` to see the details.
+///
+/// ModuleDependencyScanner should only be used via ScanningAllProjectModules.
+///
+/// The ModuleDependencyScanner can get the directly required module name for a
+/// specific source file. Also the ModuleDependencyScanner can get the source
+/// file declaring a specific module name.
+///
+/// IMPORTANT NOTE: we assume that every module unit is only declared once in a
+/// source file in the project. But the assumption is not strictly true even
+/// besides the invalid projects. The language specification requires that every
+/// module unit should be unique in a valid program. But a project can contain
+/// multiple programs. Then it is valid that we can have multiple source files
+/// declaring the same module in a project as long as these source files don't
+/// interere with each other.`
+class ModuleDependencyScanner {
+public:
+ ModuleDependencyScanner(const GlobalCompilationDatabase &CDB,
+ const ThreadsafeFS &TFS)
+ : CDB(CDB), TFS(TFS),
+ Service(tooling::dependencies::ScanningMode::CanonicalPreprocessing,
+ tooling::dependencies::ScanningOutputFormat::P1689) {}
+
+ // The scanned modules dependency information for a specific source file.
+ struct ModuleDependencyInfo {
+ // The name of the module if the file is a module unit.
+ std::optional<std::string> ModuleName;
+ // A list of names for the modules that the file directly depends.
+ std::vector<std::string> RequiredModules;
+ };
+
+ /// Scanning the single file specified by \param FilePath.
+ /// NOTE: This is only used by unittests for external uses.
+ std::optional<ModuleDependencyInfo> scan(PathRef FilePath);
+
+ /// Scanning every source file in the current project to get the
+ /// <module-name> to <module-unit-source> map.
+ /// It looks unefficiency to scan the whole project especially for
+ /// every version of every file!
+ /// TODO: We should find an efficient method to get the <module-name>
+ /// to <module-unit-source> map. We can make it either by providing
+ /// a global module dependency scanner to monitor every file. Or we
+ /// can simply require the build systems (or even if the end users)
+ /// to provide the map.
+ void globalScan(const std::vector<std::string> &AllFiles);
+ bool isGlobalScanned() const { return GlobalScanned; }
+
+ /// Get the source file from the module name. Note that the language
+ /// guarantees all the module names are unique in a valid program.
+ /// This function should only be called after globalScan.
+ ///
+ /// FIXME: We should handle the case that there are multiple source files
+ /// declaring the same module.
+ PathRef getSourceForModuleName(StringRef ModuleName) const;
+
+ /// Return the direct required modules. Indirect required modules are not
+ /// included.
+ std::vector<std::string> getRequiredModules(PathRef File);
+
+private:
+ const GlobalCompilationDatabase &CDB;
+ const ThreadsafeFS &TFS;
+
+ // Whether the scanner has scanned the project globally.
+ bool GlobalScanned = false;
+
+ clang::tooling::dependencies::DependencyScanningService Service;
+
+ // TODO: Add a scanning cache.
+
+ // Map module name to source file path.
+ llvm::StringMap<std::string> ModuleNameToSource;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/ModulesBuilder.cpp b/clang-tools-extra/clangd/ModulesBuilder.cpp
new file mode 100644
index 00000000000000..5c56f2fc31d399
--- /dev/null
+++ b/clang-tools-extra/clangd/ModulesBuilder.cpp
@@ -0,0 +1,339 @@
+//===----------------- ModulesBuilder.cpp ------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ModulesBuilder.h"
+#include "PrerequisiteModules.h"
+#include "support/Logger.h"
+
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/Frontend/FrontendActions.h"
+
+namespace clang {
+namespace clangd {
+
+namespace {
+
+/// Get or create a path to store module files. Generally it should be:
+///
+/// project_root/.cache/clangd/module_files/{RequiredPrefixDir}/.
+///
+/// \param MainFile is used to get the root of the project from global
+/// compilation database. \param RequiredPrefixDir is used to get the user
+/// defined prefix for module files. This is useful when we want to seperate
+/// module files. e.g., we want to build module files for the same module unit
+/// `a.cppm` with 2 different users `b.cpp` and `c.cpp` and we don't want the
+/// module file for `b.cpp` be conflict with the module files for `c.cpp`. Then
+/// we can put the 2 module files into different dirs like:
+///
+/// project_root/.cache/clangd/module_files/b.cpp/a.pcm
+/// project_root/.cache/clangd/module_files/c.cpp/a.pcm
+llvm::SmallString<256> getModuleFilesPath(PathRef MainFile,
+ const GlobalCompilationDatabase &CDB,
+ StringRef RequiredPrefixDir) {
+ std::optional<ProjectInfo> PI = CDB.getProjectInfo(MainFile);
+ if (!PI)
+ return {};
+
+ // FIXME: PI->SourceRoot may be empty, depending on the CDB strategy.
+ llvm::SmallString<256> Result(PI->SourceRoot);
+
+ llvm::sys::path::append(Result, ".cache");
+ llvm::sys::path::append(Result, "clangd");
+ llvm::sys::path::append(Result, "module_files");
+
+ llvm::sys::path::append(Result, RequiredPrefixDir);
+
+ llvm::sys::fs::create_directories(Result, /*IgnoreExisting=*/true);
+
+ return Result;
+}
+
+/// Get the absolute path for the filename from the compile command.
+llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) {
+ llvm::SmallString<128> AbsolutePath;
+ if (llvm::sys::path::is_absolute(Cmd.Filename)) {
+ AbsolutePath = Cmd.Filename;
+ } else {
+ AbsolutePath = Cmd.Directory;
+ llvm::sys::path::append(AbsolutePath, Cmd.Filename);
+ llvm::sys::path::remove_dots(AbsolutePath, true);
+ }
+ return AbsolutePath;
+}
+
+/// Get a unique module file path under \param ModuleFilesPrefix.
+std::string getUniqueModuleFilePath(StringRef ModuleName,
+ PathRef ModuleFilesPrefix) {
+ llvm::SmallString<256> ModuleFilePattern(ModuleFilesPrefix);
+ auto [PrimaryModuleName, PartitionName] = ModuleName.split(':');
+ llvm::sys::path::append(ModuleFilePattern, PrimaryModuleName);
+ if (!PartitionName.empty()) {
+ ModuleFilePattern.append("-");
+ ModuleFilePattern.append(PartitionName);
+ }
+
+ ModuleFilePattern.append("-%%-%%-%%-%%-%%-%%");
+ ModuleFilePattern.append(".pcm");
+
+ llvm::SmallString<256> ModuleFilePath;
+ llvm::sys::fs::createUniquePath(ModuleFilePattern, ModuleFilePath,
+ /*MakeAbsolute=*/false);
+
+ return (std::string)ModuleFilePath;
+}
+} // namespace
+
+bool ModulesBuilder::buildModuleFile(StringRef ModuleName,
+ const ThreadsafeFS *TFS,
+ std::shared_ptr<ProjectModules> MDB,
+ PathRef ModuleFilesPrefix,
+ PrerequisiteModules &BuiltModuleFiles) {
+ if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
+ return true;
+
+ PathRef ModuleUnitFileName = MDB->getSourceForModuleName(ModuleName);
+ /// It is possible that we're meeting third party modules (modules whose
+ /// source are not in the project. e.g, the std module may be a third-party
+ /// module for most project) or something wrong with the implementation of
+ /// ProjectModules.
+ /// FIXME: How should we treat third party modules here? If we want to ignore
+ /// third party modules, we should return true instead of false here.
+ /// Currently we simply bail out.
+ if (ModuleUnitFileName.empty())
+ return false;
+
+ for (auto &RequiredModuleName : MDB->getRequiredModules(ModuleUnitFileName)) {
+ // Return early if there are errors building the module file.
+ if (!buildModuleFile(RequiredModuleName, TFS, MDB, ModuleFilesPrefix,
+ BuiltModuleFiles)) {
+ log("Failed to build module {0}", RequiredModuleName);
+ return false;
+ }
+ }
+
+ auto Cmd = CDB.getCompileCommand(ModuleUnitFileName);
+ if (!Cmd)
+ return false;
+
+ std::string ModuleFileName =
+ getUniqueModuleFilePath(ModuleName, ModuleFilesPrefix);
+ Cmd->Output = ModuleFileName;
+
+ ParseInputs Inputs;
+ Inputs.TFS = TFS;
+ Inputs.CompileCommand = std::move(*Cmd);
+
+ IgnoreDiagnostics IgnoreDiags;
+ auto CI = buildCompilerInvocation(Inputs, IgnoreDiags);
+ if (!CI)
+ return false;
+
+ auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
+ auto AbsolutePath = getAbsolutePath(Inputs.CompileCommand);
+ auto Buf = FS->getBufferForFile(AbsolutePath);
+ if (!Buf)
+ return false;
+
+ // Hash the contents of input files and store the hash value to the BMI files.
+ // So that we can check if the files are still valid when we want to reuse the
+ // BMI files.
+ CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true;
+
+ BuiltModuleFiles.adjustHeaderSearchOptions(CI->getHeaderSearchOpts());
+
+ CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output;
+ auto Clang =
+ prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr,
+ std::move(*Buf), std::move(FS), IgnoreDiags);
+ if (!Clang)
+ return false;
+
+ GenerateModuleInterfaceAction Action;
+ Clang->ExecuteAction(Action);
+
+ if (Clang->getDiagnostics().hasErrorOccurred())
+ return false;
+
+ BuiltModuleFiles.addModuleFile(ModuleName, ModuleFileName);
+ return true;
+}
+
+/// FailedPrerequisiteModules - stands for the PrerequisiteModules which has
+/// errors happened during the building process.
+class FailedPrerequisiteModules : public PrerequisiteModules {
+public:
+ ~FailedPrerequisiteModules() override = default;
+
+ /// We shouldn't adjust the compilation commands based on
+ /// FailedPrerequisiteModules.
+ void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
+ }
+
+ /// FailedPrerequisiteModules can never be reused.
+ bool
+ canReuse(const CompilerInvocation &CI,
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override {
+ return false;
+ }
+
+ /// No module unit got built in FailedPrerequisiteModules.
+ bool isModuleUnitBuilt(StringRef ModuleName) const override { return false; }
+
+ /// We shouldn't add any module files to the FailedPrerequisiteModules.
+ void addModuleFile(StringRef ModuleName, StringRef ModuleFilePath) override {
+ assert(false && "We shouldn't operator based on failed module files");
+ }
+};
+
+/// StandalonePrerequisiteModules - stands for PrerequisiteModules for which all
+/// the required modules are built successfully. All the module files
+/// are owned by the StandalonePrerequisiteModules class.
+///
+/// All the built module files won't be shared with other instances of the
+/// class. So that we can avoid worrying thread safety.
+///
+/// We don't need to worry about duplicated module names here since the standard
+/// guarantees the module names should be unique to a program.
+class StandalonePrerequisiteModules : public PrerequisiteModules {
+public:
+ StandalonePrerequisiteModules() = default;
+
+ StandalonePrerequisiteModules(const StandalonePrerequisiteModules &) = delete;
+ StandalonePrerequisiteModules
+ operator=(const StandalonePrerequisiteModules &) = delete;
+ StandalonePrerequisiteModules(StandalonePrerequisiteModules &&) = delete;
+ StandalonePrerequisiteModules
+ operator=(StandalonePrerequisiteModules &&) = delete;
+
+ ~StandalonePrerequisiteModules() override = default;
+
+ void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
+ // Appending all built module files.
+ for (auto &RequiredModule : RequiredModules)
+ Options.PrebuiltModuleFiles.insert_or_assign(
+ RequiredModule.ModuleName, RequiredModule.ModuleFilePath);
+ }
+
+ bool canReuse(const CompilerInvocation &CI,
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override;
+
+ bool isModuleUnitBuilt(StringRef ModuleName) const override {
+ constexpr unsigned SmallSizeThreshold = 8;
+ if (RequiredModules.size() < SmallSizeThreshold)
+ return llvm::any_of(RequiredModules, [&](auto &MF) {
+ return MF.ModuleName == ModuleName;
+ });
+
+ return BuiltModuleNames.contains(ModuleName);
+ }
+
+ void addModuleFile(StringRef ModuleName, StringRef ModuleFilePath) override {
+ RequiredModules.emplace_back(ModuleName, ModuleFilePath);
+ BuiltModuleNames.insert(ModuleName);
+ }
+
+private:
+ struct ModuleFile {
+ ModuleFile(StringRef ModuleName, PathRef ModuleFilePath)
+ : ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()) {}
+
+ ModuleFile() = delete;
+
+ ModuleFile(const ModuleFile &) = delete;
+ ModuleFile operator=(const ModuleFile &) = delete;
+
+ // The move constructor is needed for llvm::SmallVector.
+ ModuleFile(ModuleFile &&Other)
+ : ModuleName(std::move(Other.ModuleName)),
+ ModuleFilePath(std::move(Other.ModuleFilePath)) {}
+
+ ModuleFile &operator=(ModuleFile &&Other) = delete;
+
+ ~ModuleFile() {
+ if (!ModuleFilePath.empty())
+ llvm::sys::fs::remove(ModuleFilePath);
+ }
+
+ std::string ModuleName;
+ std::string ModuleFilePath;
+ };
+
+ llvm::SmallVector<ModuleFile, 8> RequiredModules;
+ /// A helper class to speedup the query if a module is built.
+ llvm::StringSet<> BuiltModuleNames;
+};
+
+std::unique_ptr<PrerequisiteModules>
+ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
+ const ThreadsafeFS *TFS) {
+ std::shared_ptr<ProjectModules> MDB = CDB.getProjectModules(File);
+ if (!MDB)
+ return {};
+
+ std::vector<std::string> RequiredModuleNames = MDB->getRequiredModules(File);
+ if (RequiredModuleNames.empty())
+ return {};
+
+ llvm::SmallString<256> ModuleFilesPrefix =
+ getModuleFilesPath(File, CDB, llvm::sys::path::filename(File));
+
+ auto RequiredModules = std::make_unique<StandalonePrerequisiteModules>();
+
+ for (const std::string &RequiredModuleName : RequiredModuleNames)
+ // Return early if there is any error.
+ if (!buildModuleFile(RequiredModuleName, TFS, MDB, ModuleFilesPrefix,
+ *RequiredModules.get())) {
+ log("Failed to build module {0}", RequiredModuleName);
+ return std::make_unique<FailedPrerequisiteModules>();
+ }
+
+ return std::move(RequiredModules);
+}
+
+bool StandalonePrerequisiteModules::canReuse(
+ const CompilerInvocation &CI,
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) const {
+ CompilerInstance Clang;
+
+ Clang.setInvocation(std::make_shared<CompilerInvocation>(CI));
+ IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
+ CompilerInstance::createDiagnostics(new DiagnosticOptions());
+ Clang.setDiagnostics(Diags.get());
+
+ FileManager *FM = Clang.createFileManager(VFS);
+ Clang.createSourceManager(*FM);
+
+ if (!Clang.createTarget())
+ return false;
+
+ assert(Clang.getHeaderSearchOptsPtr());
+ adjustHeaderSearchOptions(Clang.getHeaderSearchOpts());
+ // Since we don't need to compile the source code actually, the TU kind here
+ // doesn't matter.
+ Clang.createPreprocessor(TU_Complete);
+ Clang.getHeaderSearchOpts().ForceCheckCXX20ModulesInputFiles = true;
+ Clang.getHeaderSearchOpts().ValidateASTInputFilesContent = true;
+
+ Clang.createASTReader();
+ for (auto &RequiredModule : RequiredModules) {
+ StringRef BMIPath = RequiredModule.ModuleFilePath;
+ auto ReadResult =
+ Clang.getASTReader()->ReadAST(BMIPath, serialization::MK_MainFile,
+ SourceLocation(), ASTReader::ARR_None);
+
+ if (ReadResult != ASTReader::Success) {
+ log("Failed to reuse {0}", BMIPath);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/ModulesBuilder.h b/clang-tools-extra/clangd/ModulesBuilder.h
new file mode 100644
index 00000000000000..ae7fc8778be676
--- /dev/null
+++ b/clang-tools-extra/clangd/ModulesBuilder.h
@@ -0,0 +1,71 @@
+//===----------------- ModulesBuilder.h --------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Experimental support for C++20 Modules.
+//
+// Currently we simplify the implementations by preventing reusing module files
+// across different versions and different source files. But this is clearly a
+// waste of time and space in the end of the day.
+//
+// FIXME: Supporting reusing module files across different versions and
+// different source files.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULES_BUILDER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULES_BUILDER_H
+
+#include "GlobalCompilationDatabase.h"
+#include "ProjectModules.h"
+
+#include "support/Path.h"
+#include "support/ThreadsafeFS.h"
+
+#include "llvm/ADT/SmallString.h"
+
+#include <memory>
+
+namespace clang {
+namespace clangd {
+
+class PrerequisiteModules;
+
+/// This class handles building module files for a given source file.
+///
+/// In the future, we want the class to manage the module files acorss
+/// different versions and different source files.
+class ModulesBuilder {
+public:
+ ModulesBuilder() = delete;
+
+ ModulesBuilder(const GlobalCompilationDatabase &CDB) : CDB(CDB) {}
+
+ ModulesBuilder(const ModulesBuilder &) = delete;
+ ModulesBuilder(ModulesBuilder &&) = delete;
+
+ ModulesBuilder &operator=(const ModulesBuilder &) = delete;
+ ModulesBuilder &operator=(ModulesBuilder &&) = delete;
+
+ ~ModulesBuilder() = default;
+
+ std::unique_ptr<PrerequisiteModules>
+ buildPrerequisiteModulesFor(PathRef File, const ThreadsafeFS *TFS);
+
+private:
+ bool buildModuleFile(StringRef ModuleName, const ThreadsafeFS *TFS,
+ std::shared_ptr<ProjectModules> MDB,
+ PathRef ModuleFilesPrefix,
+ PrerequisiteModules &RequiredModules);
+
+ const GlobalCompilationDatabase &CDB;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index edd0f77b1031ef..565f24df188fc4 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -444,6 +444,12 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
L->sawDiagnostic(D, Diag);
});
+ // Adjust header search options to load the built module files recorded
+ // in RequiredModules.
+ if (Preamble && Preamble->RequiredModules)
+ Preamble->RequiredModules->adjustHeaderSearchOptions(
+ CI->getHeaderSearchOpts());
+
std::optional<PreamblePatch> Patch;
// We might use an ignoring diagnostic consumer if they are going to be
// dropped later on to not pay for extra latency by processing them.
@@ -457,6 +463,7 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
std::move(CI), PreamblePCH,
llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, Filename), VFS,
*DiagConsumer);
+
if (!Clang) {
// The last diagnostic contains information about the reason of this
// failure.
diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index f181c7befec156..989f839fe36377 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -587,11 +587,10 @@ class DiagPatcher {
};
} // namespace
-std::shared_ptr<const PreambleData>
-buildPreamble(PathRef FileName, CompilerInvocation CI,
- const ParseInputs &Inputs, bool StoreInMemory,
- PreambleParsedCallback PreambleCallback,
- PreambleBuildStats *Stats) {
+std::shared_ptr<const PreambleData> buildPreamble(
+ PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs,
+ bool StoreInMemory, ModulesBuilder *RequiredModuleBuilder,
+ PreambleParsedCallback PreambleCallback, PreambleBuildStats *Stats) {
// Note that we don't need to copy the input contents, preamble can live
// without those.
auto ContentsBuffer =
@@ -660,10 +659,12 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
WallTimer PreambleTimer;
PreambleTimer.startTimer();
+
auto BuiltPreamble = PrecompiledPreamble::Build(
CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
Stats ? TimedFS : StatCacheFS, std::make_shared<PCHContainerOperations>(),
StoreInMemory, /*StoragePath=*/"", CapturedInfo);
+
PreambleTimer.stopTimer();
// We have to setup DiagnosticConsumer that will be alife
@@ -696,6 +697,19 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
Result->Includes = CapturedInfo.takeIncludes();
Result->Pragmas = std::make_shared<const include_cleaner::PragmaIncludes>(
CapturedInfo.takePragmaIncludes());
+
+ if (RequiredModuleBuilder) {
+ WallTimer PrerequisiteModuleTimer;
+ PrerequisiteModuleTimer.startTimer();
+ Result->RequiredModules =
+ RequiredModuleBuilder->buildPrerequisiteModulesFor(FileName,
+ Inputs.TFS);
+ PrerequisiteModuleTimer.stopTimer();
+
+ log("Built prerequisite modules for file {0} in {1} seconds", FileName,
+ PrerequisiteModuleTimer.getTime());
+ }
+
Result->Macros = CapturedInfo.takeMacros();
Result->Marks = CapturedInfo.takeMarks();
Result->StatCache = StatCache;
@@ -736,7 +750,9 @@ bool isPreambleCompatible(const PreambleData &Preamble,
auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
return compileCommandsAreEqual(Inputs.CompileCommand,
Preamble.CompileCommand) &&
- Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS);
+ Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS) &&
+ (!Preamble.RequiredModules ||
+ Preamble.RequiredModules->canReuse(CI, VFS));
}
void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) {
diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h
index 37da3833748a9c..076677ab3bed5f 100644
--- a/clang-tools-extra/clangd/Preamble.h
+++ b/clang-tools-extra/clangd/Preamble.h
@@ -27,6 +27,10 @@
#include "Diagnostics.h"
#include "FS.h"
#include "Headers.h"
+
+#include "ModulesBuilder.h"
+#include "PrerequisiteModules.h"
+
#include "clang-include-cleaner/Record.h"
#include "support/Path.h"
#include "clang/Basic/SourceManager.h"
@@ -104,6 +108,8 @@ struct PreambleData {
IncludeStructure Includes;
// Captures #include-mapping information in #included headers.
std::shared_ptr<const include_cleaner::PragmaIncludes> Pragmas;
+ // Information about required module files for this preamble.
+ std::unique_ptr<PrerequisiteModules> RequiredModules;
// Macros defined in the preamble section of the main file.
// Users care about headers vs main-file, not preamble vs non-preamble.
// These should be treated as main-file entities e.g. for code completion.
@@ -144,9 +150,13 @@ struct PreambleBuildStats {
/// If \p PreambleCallback is set, it will be run on top of the AST while
/// building the preamble.
/// If Stats is not non-null, build statistics will be exported there.
+/// If \p RequiredModuleBuilder is not null, it will scan the source file
+/// to see if it is related to modules, and if yes, modules related things
+/// will be built.
std::shared_ptr<const PreambleData>
buildPreamble(PathRef FileName, CompilerInvocation CI,
const ParseInputs &Inputs, bool StoreInMemory,
+ ModulesBuilder *RequiredModuleBuilder,
PreambleParsedCallback PreambleCallback,
PreambleBuildStats *Stats = nullptr);
diff --git a/clang-tools-extra/clangd/PrerequisiteModules.h b/clang-tools-extra/clangd/PrerequisiteModules.h
new file mode 100644
index 00000000000000..a73beb76480248
--- /dev/null
+++ b/clang-tools-extra/clangd/PrerequisiteModules.h
@@ -0,0 +1,87 @@
+//===----------------- PrerequisiteModules.h ---------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREREQUISITEMODULES_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREREQUISITEMODULES_H
+
+#include "Compiler.h"
+#include "support/Path.h"
+
+#include "clang/Lex/HeaderSearchOptions.h"
+
+#include "llvm/ADT/StringSet.h"
+
+namespace clang {
+namespace clangd {
+
+class ModulesBuilder;
+
+/// Store all the needed module files information to parse a single
+/// source file. e.g.,
+///
+/// ```
+/// // a.cppm
+/// export module a;
+///
+/// // b.cppm
+/// export module b;
+/// import a;
+///
+/// // c.cppm
+/// export module c;
+/// import a;
+/// ```
+///
+/// For the source file `c.cppm`, an instance of the class will store
+/// the module files for `a.cppm` and `b.cppm`. But the module file for `c.cppm`
+/// won't be stored. Since it is not needed to parse `c.cppm`.
+///
+/// Users should only get PrerequisiteModules from
+/// `ModulesBuilder::buildPrerequisiteModulesFor(...)`.
+///
+/// Users can detect whether the PrerequisiteModules is still up to date by
+/// calling the `canReuse()` member function.
+///
+/// The users should call `adjustHeaderSearchOptions(...)` to update the
+/// compilation commands to select the built module files first. Before calling
+/// `adjustHeaderSearchOptions()`, users should call `canReuse()` first to check
+/// if all the stored module files are valid. In case they are not valid,
+/// users should call `ModulesBuilder::buildPrerequisiteModulesFor(...)` again
+/// to get the new PrerequisiteModules.
+class PrerequisiteModules {
+public:
+ /// Change commands to load the module files recorded in this
+ /// PrerequisiteModules first.
+ virtual void
+ adjustHeaderSearchOptions(HeaderSearchOptions &Options) const = 0;
+
+ /// Whether or not the built module files are up to date.
+ /// Note that this can only be used after building the module files.
+ virtual bool
+ canReuse(const CompilerInvocation &CI,
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const = 0;
+
+ /// Return true if the modile file specified by ModuleName is built.
+ /// Note that this interface will only check the existence of the module
+ /// file instead of checking the validness of the module file.
+ virtual bool isModuleUnitBuilt(StringRef ModuleName) const = 0;
+
+ virtual ~PrerequisiteModules() = default;
+
+private:
+ friend class ModulesBuilder;
+
+ /// Add a module file to the PrerequisiteModules.
+ virtual void addModuleFile(StringRef ModuleName,
+ StringRef ModuleFilePath) = 0;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/ProjectModules.cpp b/clang-tools-extra/clangd/ProjectModules.cpp
new file mode 100644
index 00000000000000..6a9d9731c309ae
--- /dev/null
+++ b/clang-tools-extra/clangd/ProjectModules.cpp
@@ -0,0 +1,62 @@
+//===------------------ ProjectModules.h -------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ProjectModules.h"
+
+namespace clang {
+namespace clangd {
+
+/// TODO: The existing `ScanningAllProjectModules` is not efficient. See the
+/// comments in ModuleDependencyScanner for detail.
+///
+/// In the future, we wish the build system can provide a well design
+/// compilation database for modules then we can query that new compilation
+/// database directly. Or we need to have a global long-live scanner to detect
+/// the state of each file.
+class ScanningAllProjectModules : public ProjectModules {
+public:
+ ScanningAllProjectModules(std::vector<std::string> &&AllFiles,
+ const GlobalCompilationDatabase &CDB,
+ const ThreadsafeFS &TFS)
+ : AllFiles(std::move(AllFiles)), Scanner(CDB, TFS) {}
+
+ ~ScanningAllProjectModules() override = default;
+
+ std::vector<std::string> getRequiredModules(PathRef File) override {
+ return Scanner.getRequiredModules(File);
+ }
+
+ /// RequiredSourceFile is not used intentionally. See the comments of
+ /// ModuleDependencyScanner for detail.
+ PathRef
+ getSourceForModuleName(StringRef ModuleName,
+ PathRef RequiredSourceFile = PathRef()) override {
+ if (!Scanner.isGlobalScanned())
+ Scanner.globalScan(AllFiles);
+
+ return Scanner.getSourceForModuleName(ModuleName);
+ }
+
+private:
+ std::vector<std::string> AllFiles;
+
+ ModuleDependencyScanner Scanner;
+};
+
+std::shared_ptr<ProjectModules> ProjectModules::create(
+ ProjectModulesKind Kind, std::vector<std::string> &&AllFiles,
+ const GlobalCompilationDatabase &CDB, const ThreadsafeFS &TFS) {
+ if (Kind == ProjectModulesKind::ScanningAllFiles)
+ return std::make_shared<ScanningAllProjectModules>(std::move(AllFiles), CDB,
+ TFS);
+
+ llvm_unreachable("Unknown ProjectModulesKind.");
+}
+
+} // namespace clangd
+} // namespace clang
\ No newline at end of file
diff --git a/clang-tools-extra/clangd/ProjectModules.h b/clang-tools-extra/clangd/ProjectModules.h
new file mode 100644
index 00000000000000..98720ee06d4724
--- /dev/null
+++ b/clang-tools-extra/clangd/ProjectModules.h
@@ -0,0 +1,55 @@
+//===------------------ ProjectModules.h -------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PROJECTMODULES_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PROJECTMODULES_H
+
+#include "ModuleDependencyScanner.h"
+
+#include <memory>
+
+namespace clang {
+namespace clangd {
+
+/// An interface to query the modules information in the project.
+/// Users should get instances of `ProjectModules` from
+/// `GlobalCompilationDatabase::getProjectModules(PathRef)`.
+///
+/// Currently, the modules information includes:
+/// - Given a source file, what are the required modules.
+/// - Given a module name and a required source file, what is
+/// the corresponding source file.
+///
+/// Note that there can be multiple source files declaring the same module
+/// in a valid project. Although the language specification requires that
+/// every module unit's name must be unique in valid program, there can be
+/// multiple program in a project. And it is technically valid if these program
+/// doesn't interfere with each other.
+///
+/// A module name should be in the format:
+/// `<primary-module-name>[:partition-name]`. So module names covers partitions.
+class ProjectModules {
+public:
+ enum class ProjectModulesKind { ScanningAllFiles };
+
+ static std::shared_ptr<ProjectModules>
+ create(ProjectModulesKind Kind, std::vector<std::string> &&AllFiles,
+ const GlobalCompilationDatabase &CDB, const ThreadsafeFS &TFS);
+
+ virtual std::vector<std::string> getRequiredModules(PathRef File) = 0;
+ virtual PathRef
+ getSourceForModuleName(StringRef ModuleName,
+ PathRef RequiredSrcFile = PathRef()) = 0;
+
+ virtual ~ProjectModules() = default;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp
index 324ba1fc8cb895..cb90d29c2313ae 100644
--- a/clang-tools-extra/clangd/TUScheduler.cpp
+++ b/clang-tools-extra/clangd/TUScheduler.cpp
@@ -52,6 +52,7 @@
#include "Config.h"
#include "Diagnostics.h"
#include "GlobalCompilationDatabase.h"
+#include "ModulesBuilder.h"
#include "ParsedAST.h"
#include "Preamble.h"
#include "clang-include-cleaner/Record.h"
@@ -605,8 +606,8 @@ class ASTWorker {
ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &LRUCache,
TUScheduler::HeaderIncluderCache &HeaderIncluders,
- Semaphore &Barrier, bool RunSync, const TUScheduler::Options &Opts,
- ParsingCallbacks &Callbacks);
+ Semaphore &Barrier, ModulesBuilder *ModulesManager, bool RunSync,
+ const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks);
public:
/// Create a new ASTWorker and return a handle to it.
@@ -619,7 +620,8 @@ class ASTWorker {
TUScheduler::ASTCache &IdleASTs,
TUScheduler::HeaderIncluderCache &HeaderIncluders,
AsyncTaskRunner *Tasks, Semaphore &Barrier,
- const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks);
+ ModulesBuilder *ModulesManager, const TUScheduler::Options &Opts,
+ ParsingCallbacks &Callbacks);
~ASTWorker();
void update(ParseInputs Inputs, WantDiagnostics, bool ContentChanged);
@@ -652,6 +654,10 @@ class ASTWorker {
TUScheduler::FileStats stats() const;
bool isASTCached() const;
+ const GlobalCompilationDatabase &getCompilationDatabase() { return CDB; }
+
+ ModulesBuilder *getModulesManager() const { return ModulesManager; }
+
private:
// Details of an update request that are relevant to scheduling.
struct UpdateType {
@@ -710,6 +716,7 @@ class ASTWorker {
TUScheduler::ASTCache &IdleASTs;
TUScheduler::HeaderIncluderCache &HeaderIncluders;
const bool RunSync;
+
/// Time to wait after an update to see whether another update obsoletes it.
const DebouncePolicy UpdateDebounce;
/// File that ASTWorker is responsible for.
@@ -763,6 +770,8 @@ class ASTWorker {
SynchronizedTUStatus Status;
PreambleThread PreamblePeer;
+
+ ModulesBuilder *ModulesManager = nullptr;
};
/// A smart-pointer-like class that points to an active ASTWorker.
@@ -807,16 +816,15 @@ class ASTWorkerHandle {
std::shared_ptr<ASTWorker> Worker;
};
-ASTWorkerHandle
-ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
- TUScheduler::ASTCache &IdleASTs,
- TUScheduler::HeaderIncluderCache &HeaderIncluders,
- AsyncTaskRunner *Tasks, Semaphore &Barrier,
- const TUScheduler::Options &Opts,
- ParsingCallbacks &Callbacks) {
- std::shared_ptr<ASTWorker> Worker(
- new ASTWorker(FileName, CDB, IdleASTs, HeaderIncluders, Barrier,
- /*RunSync=*/!Tasks, Opts, Callbacks));
+ASTWorkerHandle ASTWorker::create(
+ PathRef FileName, const GlobalCompilationDatabase &CDB,
+ TUScheduler::ASTCache &IdleASTs,
+ TUScheduler::HeaderIncluderCache &HeaderIncluders, AsyncTaskRunner *Tasks,
+ Semaphore &Barrier, ModulesBuilder *ModulesManager,
+ const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks) {
+ std::shared_ptr<ASTWorker> Worker(new ASTWorker(
+ FileName, CDB, IdleASTs, HeaderIncluders, Barrier, ModulesManager,
+ /*RunSync=*/!Tasks, Opts, Callbacks));
if (Tasks) {
Tasks->runAsync("ASTWorker:" + llvm::sys::path::filename(FileName),
[Worker]() { Worker->run(); });
@@ -830,15 +838,16 @@ ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &LRUCache,
TUScheduler::HeaderIncluderCache &HeaderIncluders,
- Semaphore &Barrier, bool RunSync,
- const TUScheduler::Options &Opts,
+ Semaphore &Barrier, ModulesBuilder *ModulesManager,
+ bool RunSync, const TUScheduler::Options &Opts,
ParsingCallbacks &Callbacks)
: IdleASTs(LRUCache), HeaderIncluders(HeaderIncluders), RunSync(RunSync),
UpdateDebounce(Opts.UpdateDebounce), FileName(FileName),
ContextProvider(Opts.ContextProvider), CDB(CDB), Callbacks(Callbacks),
Barrier(Barrier), Done(false), Status(FileName, Callbacks),
PreamblePeer(FileName, Callbacks, Opts.StorePreamblesInMemory, RunSync,
- Opts.PreambleThrottler, Status, HeaderIncluders, *this) {
+ Opts.PreambleThrottler, Status, HeaderIncluders, *this),
+ ModulesManager(ModulesManager) {
// Set a fallback command because compile command can be accessed before
// `Inputs` is initialized. Other fields are only used after initialization
// from client inputs.
@@ -1082,8 +1091,9 @@ void PreambleThread::build(Request Req) {
PreambleBuildStats Stats;
bool IsFirstPreamble = !LatestBuild;
+
LatestBuild = clang::clangd::buildPreamble(
- FileName, *Req.CI, Inputs, StoreInMemory,
+ FileName, *Req.CI, Inputs, StoreInMemory, ASTPeer.getModulesManager(),
[&](CapturedASTCtx ASTCtx,
std::shared_ptr<const include_cleaner::PragmaIncludes> PI) {
Callbacks.onPreambleAST(FileName, Inputs.Version, std::move(ASTCtx),
@@ -1644,6 +1654,9 @@ TUScheduler::TUScheduler(const GlobalCompilationDatabase &CDB,
PreambleTasks.emplace();
WorkerThreads.emplace();
}
+
+ if (Opts.ExperimentalModulesSupport)
+ ModulesManager.emplace(CDB);
}
TUScheduler::~TUScheduler() {
@@ -1676,7 +1689,8 @@ bool TUScheduler::update(PathRef File, ParseInputs Inputs,
// Create a new worker to process the AST-related tasks.
ASTWorkerHandle Worker = ASTWorker::create(
File, CDB, *IdleASTs, *HeaderIncluders,
- WorkerThreads ? &*WorkerThreads : nullptr, Barrier, Opts, *Callbacks);
+ WorkerThreads ? &*WorkerThreads : nullptr, Barrier,
+ ModulesManager ? &*ModulesManager : nullptr, Opts, *Callbacks);
FD = std::unique_ptr<FileData>(
new FileData{Inputs.Contents, std::move(Worker)});
ContentChanged = true;
diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h
index fb936d46bbcf7e..66e17cbaa7f032 100644
--- a/clang-tools-extra/clangd/TUScheduler.h
+++ b/clang-tools-extra/clangd/TUScheduler.h
@@ -204,6 +204,8 @@ class ParsingCallbacks {
virtual void onPreamblePublished(PathRef File) {}
};
+class ModulesBuilder;
+
/// Handles running tasks for ClangdServer and managing the resources (e.g.,
/// preambles and ASTs) for opened files.
/// TUScheduler is not thread-safe, only one thread should be providing updates
@@ -222,6 +224,9 @@ class TUScheduler {
/// Cache (large) preamble data in RAM rather than temporary files on disk.
bool StorePreamblesInMemory = false;
+ /// Enable experimental support for modules.
+ bool ExperimentalModulesSupport = false;
+
/// Time to wait after an update to see if another one comes along.
/// This tries to ensure we rebuild once the user stops typing.
DebouncePolicy UpdateDebounce;
@@ -372,6 +377,8 @@ class TUScheduler {
// running tasks asynchronously.
std::optional<AsyncTaskRunner> PreambleTasks;
std::optional<AsyncTaskRunner> WorkerThreads;
+ // Manages to build module files.
+ std::optional<ModulesBuilder> ModulesManager;
// Used to create contexts for operations that are not bound to a particular
// file (e.g. index queries).
std::string LastActiveFile;
diff --git a/clang-tools-extra/clangd/test/CMakeLists.txt b/clang-tools-extra/clangd/test/CMakeLists.txt
index d073267066e0b4..b51f461a498665 100644
--- a/clang-tools-extra/clangd/test/CMakeLists.txt
+++ b/clang-tools-extra/clangd/test/CMakeLists.txt
@@ -2,6 +2,7 @@ set(CLANGD_TEST_DEPS
clangd
ClangdTests
clangd-indexer
+ split-file
# No tests for it, but we should still make sure they build.
dexp
)
diff --git a/clang-tools-extra/clangd/test/modules.test b/clang-tools-extra/clangd/test/modules.test
new file mode 100644
index 00000000000000..74280811a6cff8
--- /dev/null
+++ b/clang-tools-extra/clangd/test/modules.test
@@ -0,0 +1,83 @@
+# A smoke test to check the modules can work basically.
+#
+# Windows have different escaping modes.
+# FIXME: We should add one for windows.
+# UNSUPPORTED: system-windows
+#
+# RUN: rm -fr %t
+# RUN: mkdir -p %t
+# RUN: split-file %s %t
+#
+# RUN: sed -e "s|DIR|%/t|g" %t/compile_commands.json.tmpl > %t/compile_commands.json.tmp
+# RUN: sed -e "s|CLANG_CC|%clang|g" %t/compile_commands.json.tmp > %t/compile_commands.json
+# RUN: sed -e "s|DIR|%/t|g" %t/definition.jsonrpc.tmpl > %t/definition.jsonrpc
+#
+# RUN: clangd -experimental-modules-support -lit-test < %t/definition.jsonrpc \
+# RUN: | FileCheck -strict-whitespace %t/definition.jsonrpc
+
+#--- A.cppm
+export module A;
+export void printA() {}
+
+#--- Use.cpp
+import A;
+void foo() {
+ print
+}
+
+#--- compile_commands.json.tmpl
+[
+ {
+ "directory": "DIR",
+ "command": "CLANG_CC -fprebuilt-module-path=DIR -std=c++20 -o DIR/main.cpp.o -c DIR/Use.cpp",
+ "file": "DIR/Use.cpp"
+ },
+ {
+ "directory": "DIR",
+ "command": "CLANG_CC -std=c++20 DIR/A.cppm --precompile -o DIR/A.pcm",
+ "file": "DIR/A.cppm"
+ }
+]
+
+#--- definition.jsonrpc.tmpl
+{
+ "jsonrpc": "2.0",
+ "id": 0,
+ "method": "initialize",
+ "params": {
+ "processId": 123,
+ "rootPath": "clangd",
+ "capabilities": {
+ "textDocument": {
+ "completion": {
+ "completionItem": {
+ "snippetSupport": true
+ }
+ }
+ }
+ },
+ "trace": "off"
+ }
+}
+---
+{
+ "jsonrpc": "2.0",
+ "method": "textDocument/didOpen",
+ "params": {
+ "textDocument": {
+ "uri": "file://DIR/Use.cpp",
+ "languageId": "cpp",
+ "version": 1,
+ "text": "import A;\nvoid foo() {\n print\n}\n"
+ }
+ }
+}
+
+# CHECK: "message"{{.*}}printA{{.*}}(fix available)
+
+---
+{"jsonrpc":"2.0","id":1,"method":"textDocument/completion","params":{"textDocument":{"uri":"file://DIR/Use.cpp"},"context":{"triggerKind":1},"position":{"line":2,"character":6}}}
+---
+{"jsonrpc":"2.0","id":2,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp
index b5c4d145619df3..1a245b4d007b8c 100644
--- a/clang-tools-extra/clangd/tool/Check.cpp
+++ b/clang-tools-extra/clangd/tool/Check.cpp
@@ -146,6 +146,8 @@ class Checker {
ClangdLSPServer::Options Opts;
// from buildCommand
tooling::CompileCommand Cmd;
+ std::unique_ptr<GlobalCompilationDatabase> BaseCDB;
+ std::unique_ptr<OverlayCDB> CDB;
// from buildInvocation
ParseInputs Inputs;
std::unique_ptr<CompilerInvocation> Invocation;
@@ -168,14 +170,14 @@ class Checker {
DirectoryBasedGlobalCompilationDatabase::Options CDBOpts(TFS);
CDBOpts.CompileCommandsDir =
Config::current().CompileFlags.CDBSearch.FixedCDBPath;
- std::unique_ptr<GlobalCompilationDatabase> BaseCDB =
+ BaseCDB =
std::make_unique<DirectoryBasedGlobalCompilationDatabase>(CDBOpts);
auto Mangler = CommandMangler::detect();
Mangler.SystemIncludeExtractor =
getSystemIncludeExtractor(llvm::ArrayRef(Opts.QueryDriverGlobs));
if (Opts.ResourceDir)
Mangler.ResourceDir = *Opts.ResourceDir;
- auto CDB = std::make_unique<OverlayCDB>(
+ CDB = std::make_unique<OverlayCDB>(
BaseCDB.get(), std::vector<std::string>{}, std::move(Mangler));
if (auto TrueCmd = CDB->getCompileCommand(File)) {
@@ -234,8 +236,15 @@ class Checker {
// Build preamble and AST, and index them.
bool buildAST() {
log("Building preamble...");
+
+ auto RequiredModuleBuilder =
+ Opts.ExperimentalModulesSupport
+ ? std::make_unique<ModulesBuilder>(*CDB.get())
+ : nullptr;
+
Preamble = buildPreamble(
File, *Invocation, Inputs, /*StoreInMemory=*/true,
+ RequiredModuleBuilder.get(),
[&](CapturedASTCtx Ctx,
std::shared_ptr<const include_cleaner::PragmaIncludes> PI) {
if (!Opts.BuildDynamicSymbolIndex)
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index f656a8c587c653..e38b7c567fb7a9 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -550,6 +550,13 @@ opt<std::string> ProjectRoot{
};
#endif
+opt<bool> ExperimentalModulesSupport{
+ "experimental-modules-support",
+ cat(Features),
+ desc("Experimental support for standard c++ modules"),
+ init(false),
+};
+
/// Supports a test URI scheme with relaxed constraints for lit tests.
/// The path in a test URI will be combined with a platform-specific fake
/// directory to form an absolute path. For example, test:///a.cpp is resolved
@@ -861,6 +868,7 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
ClangdLSPServer::Options Opts;
Opts.UseDirBasedCDB = (CompileArgsFrom == FilesystemCompileArgs);
+ Opts.ExperimentalModulesSupport = ExperimentalModulesSupport;
switch (PCHStorage) {
case PCHStorageFlag::Memory:
diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index 8d02b91fdd7166..529b8d2a11733f 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -72,6 +72,8 @@ add_unittest(ClangdUnitTests ClangdTests
LoggerTests.cpp
LSPBinderTests.cpp
LSPClient.cpp
+ ModuleDependencyScannerTest.cpp
+ PrerequisiteModulesTest.cpp
ModulesTests.cpp
ParsedASTTests.cpp
PathMappingTests.cpp
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 766998eb4f3c71..0cb14c429696b7 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -132,8 +132,12 @@ CodeCompleteResult completions(const TestTU &TU, Position Point,
ADD_FAILURE() << "Couldn't build CompilerInvocation";
return {};
}
+
+ MockCompilationDatabase CDB;
auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
- /*InMemory=*/true, /*Callback=*/nullptr);
+ /*InMemory=*/true,
+ /*RequiredModuleBuilder=*/nullptr,
+ /*Callback=*/nullptr);
return codeComplete(testPath(TU.Filename), Point, Preamble.get(), Inputs,
Opts);
}
@@ -1360,8 +1364,12 @@ signatures(llvm::StringRef Text, Position Point,
ADD_FAILURE() << "Couldn't build CompilerInvocation";
return {};
}
- auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
- /*InMemory=*/true, /*Callback=*/nullptr);
+
+ MockCompilationDatabase CDB;
+ auto Preamble =
+ buildPreamble(testPath(TU.Filename), *CI, Inputs,
+ /*InMemory=*/true, /*RequiredModuleBuilder=*/nullptr,
+ /*Callback=*/nullptr);
if (!Preamble) {
ADD_FAILURE() << "Couldn't build Preamble";
return {};
@@ -1642,8 +1650,12 @@ TEST(SignatureHelpTest, StalePreamble) {
auto Inputs = TU.inputs(FS);
auto CI = buildCompilerInvocation(Inputs, Diags);
ASSERT_TRUE(CI);
- auto EmptyPreamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
- /*InMemory=*/true, /*Callback=*/nullptr);
+
+ MockCompilationDatabase CDB;
+ auto EmptyPreamble =
+ buildPreamble(testPath(TU.Filename), *CI, Inputs,
+ /*InMemory=*/true, /*RequiredModuleBuilder=*/nullptr,
+ /*Callback=*/nullptr);
ASSERT_TRUE(EmptyPreamble);
TU.AdditionalFiles["a.h"] = "int foo(int x);";
diff --git a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
index cf30b388d234df..b3370d5f4da9b1 100644
--- a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
@@ -336,9 +336,11 @@ TEST(FileIndexTest, RebuildWithPreamble) {
FileIndex Index;
bool IndexUpdated = false;
+
+ MockCompilationDatabase CDB;
buildPreamble(
FooCpp, *CI, PI,
- /*StoreInMemory=*/true,
+ /*StoreInMemory=*/true, /*RequiredModuleBuilder=*/nullptr,
[&](CapturedASTCtx ASTCtx,
std::shared_ptr<const include_cleaner::PragmaIncludes> PI) {
auto &Ctx = ASTCtx.getASTContext();
diff --git a/clang-tools-extra/clangd/unittests/ModuleDependencyScannerTest.cpp b/clang-tools-extra/clangd/unittests/ModuleDependencyScannerTest.cpp
new file mode 100644
index 00000000000000..6d42f8730538f2
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/ModuleDependencyScannerTest.cpp
@@ -0,0 +1,176 @@
+//===------------ ModuleDependencyScannerTest.cpp ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+/// FIXME: Skip testing on windows temporarily due to the different escaping
+/// code mode.
+#ifndef _WIN32
+
+#include "ModuleDependencyScanner.h"
+#include "ModulesTestSetup.h"
+#include "TestFS.h"
+
+using namespace clang;
+using namespace clang::clangd;
+using namespace clang::tooling::dependencies;
+
+namespace {
+class ModuleDependencyScannerTests : public ModuleTestSetup {};
+
+TEST_F(ModuleDependencyScannerTests, SingleFile) {
+ addFile("foo.h", R"cpp(
+import foo;
+ )cpp");
+
+ addFile("A.cppm", R"cpp(
+module;
+#include "foo.h"
+export module A;
+export import :partA;
+import :partB;
+import C;
+
+module :private;
+import D;
+ )cpp");
+
+ MockCompilationDatabase CDB(TestDir);
+ CDB.ExtraClangFlags.push_back("-std=c++20");
+
+ ModuleDependencyScanner Scanner(CDB, TFS);
+ std::optional<ModuleDependencyScanner::ModuleDependencyInfo> ScanningResult =
+ Scanner.scan(getFullPath("A.cppm"));
+ EXPECT_TRUE(ScanningResult);
+
+ EXPECT_TRUE(ScanningResult->ModuleName);
+ EXPECT_EQ(*ScanningResult->ModuleName, "A");
+
+ EXPECT_EQ(ScanningResult->RequiredModules.size(), 5u);
+ EXPECT_EQ(ScanningResult->RequiredModules[0], "foo");
+ EXPECT_EQ(ScanningResult->RequiredModules[1], "A:partA");
+ EXPECT_EQ(ScanningResult->RequiredModules[2], "A:partB");
+ EXPECT_EQ(ScanningResult->RequiredModules[3], "C");
+ EXPECT_EQ(ScanningResult->RequiredModules[4], "D");
+}
+
+TEST_F(ModuleDependencyScannerTests, GlobalScanning) {
+ addFile("build/compile_commands.json", R"cpp(
+[
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/foo.cppm -fmodule-output=__DIR__/foo.pcm -c -o __DIR__/foo.o",
+ "file": "__DIR__/foo.cppm",
+ "output": "__DIR__/foo.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/C.cppm -fmodule-output=__DIR__/C.pcm -c -o __DIR__/C.o",
+ "file": "__DIR__/C.cppm",
+ "output": "__DIR__/C.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/D.cppm -fmodule-output=__DIR__/D.pcm -c -o __DIR__/D.o",
+ "file": "__DIR__/D.cppm",
+ "output": "__DIR__/D.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/A-partA.cppm -fmodule-file=foo=__DIR__/foo.pcm -fmodule-output=__DIR__/A-partA.pcm -c -o __DIR__/A-partA.o",
+ "file": "__DIR__/A-partA.cppm",
+ "output": "__DIR__/A-partA.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/A-partB.cppm -fmodule-file=C=__DIR__/C.pcm -fmodule-output=__DIR__/A-partB.pcm -c -o __DIR__/A-partB.o",
+ "file": "__DIR__/A-partB.cppm",
+ "output": "__DIR__/A-partB.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/A.cppm -fmodule-file=A:partB=__DIR__/A-partB.pcm -fmodule-file=A:partA=__DIR__/A-partA.pcm -fmodule-file=foo=__DIR__/foo.pcm -fmodule-file=C=__DIR__/C.pcm -fmodule-file=D=__DIR__/C.pcm -fmodule-output=__DIR__/A.pcm -c -o __DIR__/A.o",
+ "file": "__DIR__/A.cppm",
+ "output": "__DIR__/A.o"
+},
+]
+ )cpp");
+
+ addFile("foo.cppm", R"cpp(
+export module foo;
+ )cpp");
+
+ addFile("foo.h", R"cpp(
+import foo;
+ )cpp");
+
+ addFile("A-partA.cppm", R"cpp(
+export module A:partA;
+import foo;
+ )cpp");
+
+ addFile("A-partB.cppm", R"cpp(
+module A:partB;
+import C;
+ )cpp");
+
+ addFile("C.cppm", R"cpp(
+export module C;
+ )cpp");
+
+ addFile("D.cppm", R"cpp(
+export module D;
+ )cpp");
+
+ addFile("A.cppm", R"cpp(
+module;
+#include "foo.h"
+export module A;
+export import :partA;
+import :partB;
+import C;
+
+module :private;
+import D;
+ )cpp");
+
+ std::unique_ptr<GlobalCompilationDatabase> CDB =
+ getGlobalCompilationDatabase();
+ ModuleDependencyScanner Scanner(*CDB.get(), TFS);
+ Scanner.globalScan({getFullPath("A.cppm"), getFullPath("foo.cppm"),
+ getFullPath("A-partA.cppm"), getFullPath("A-partB.cppm"),
+ getFullPath("C.cppm"), getFullPath("D.cppm")});
+
+ EXPECT_TRUE(Scanner.getSourceForModuleName("foo").endswith("foo.cppm"));
+ EXPECT_TRUE(Scanner.getSourceForModuleName("A").endswith("A.cppm"));
+ EXPECT_TRUE(
+ Scanner.getSourceForModuleName("A:partA").endswith("A-partA.cppm"));
+ EXPECT_TRUE(
+ Scanner.getSourceForModuleName("A:partB").endswith("A-partB.cppm"));
+ EXPECT_TRUE(Scanner.getSourceForModuleName("C").endswith("C.cppm"));
+ EXPECT_TRUE(Scanner.getSourceForModuleName("D").endswith("D.cppm"));
+
+ EXPECT_TRUE(Scanner.getRequiredModules(getFullPath("foo.cppm")).empty());
+ EXPECT_TRUE(Scanner.getRequiredModules(getFullPath("C.cppm")).empty());
+ EXPECT_TRUE(Scanner.getRequiredModules(getFullPath("D.cppm")).empty());
+
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A-partA.cppm")).size(), 1u);
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A-partA.cppm"))[0], "foo");
+
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A-partB.cppm")).size(), 1u);
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A-partB.cppm"))[0], "C");
+
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A.cppm")).size(), 5u);
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A.cppm"))[0], "foo");
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A.cppm"))[1], "A:partA");
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A.cppm"))[2], "A:partB");
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A.cppm"))[3], "C");
+ EXPECT_EQ(Scanner.getRequiredModules(getFullPath("A.cppm"))[4], "D");
+}
+
+} // namespace
+
+#endif
diff --git a/clang-tools-extra/clangd/unittests/ModulesTestSetup.h b/clang-tools-extra/clangd/unittests/ModulesTestSetup.h
new file mode 100644
index 00000000000000..3f66c5ff80dc69
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/ModulesTestSetup.h
@@ -0,0 +1,103 @@
+//===-- ModulesTestSetup.h - Setup the module test environment --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Compiler.h"
+#include "support/ThreadsafeFS.h"
+
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+class ModuleTestSetup : public ::testing::Test {
+protected:
+ void SetUp() override {
+ ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory("modules-test", TestDir));
+ }
+
+ void TearDown() override { llvm::sys::fs::remove_directories(TestDir); }
+
+public:
+ // Add files to the working testing directory and repalce all the
+ // `__DIR__` to TestDir.
+ void addFile(StringRef Path, StringRef Contents) {
+ ASSERT_FALSE(llvm::sys::path::is_absolute(Path));
+
+ SmallString<256> AbsPath(TestDir);
+ llvm::sys::path::append(AbsPath, Path);
+
+ ASSERT_FALSE(llvm::sys::fs::create_directories(
+ llvm::sys::path::parent_path(AbsPath)));
+
+ std::error_code EC;
+ llvm::raw_fd_ostream OS(AbsPath, EC);
+ ASSERT_FALSE(EC);
+
+ std::size_t Pos = Contents.find("__DIR__");
+ while (Pos != llvm::StringRef::npos) {
+ OS << Contents.take_front(Pos);
+ OS << TestDir;
+ Contents = Contents.drop_front(Pos + sizeof("__DIR__") - 1);
+ Pos = Contents.find("__DIR__");
+ }
+
+ OS << Contents;
+ }
+
+ // Get the absolute path for file specified by Path under testing working
+ // directory.
+ std::string getFullPath(StringRef Path) {
+ SmallString<128> Result(TestDir);
+ llvm::sys::path::append(Result, Path);
+ EXPECT_TRUE(llvm::sys::fs::exists(Result.str()));
+ return Result.str().str();
+ }
+
+ std::unique_ptr<GlobalCompilationDatabase> getGlobalCompilationDatabase() {
+ // The compilation flags with modules are much complex so it looks better
+ // to use DirectoryBasedGlobalCompilationDatabase than a mocked compilation
+ // database.
+ DirectoryBasedGlobalCompilationDatabase::Options Opts(TFS);
+ return std::make_unique<DirectoryBasedGlobalCompilationDatabase>(Opts);
+ }
+
+ ParseInputs getInputs(StringRef FileName,
+ const GlobalCompilationDatabase &CDB) {
+ std::string FullPathName = getFullPath(FileName);
+
+ ParseInputs Inputs;
+ std::optional<tooling::CompileCommand> Cmd =
+ CDB.getCompileCommand(FullPathName);
+ EXPECT_TRUE(Cmd);
+ Inputs.CompileCommand = std::move(*Cmd);
+ Inputs.TFS = &TFS;
+
+ if (auto Contents = TFS.view(TestDir)->getBufferForFile(FullPathName))
+ Inputs.Contents = Contents->get()->getBuffer().str();
+
+ return Inputs;
+ }
+
+ std::unique_ptr<CompilerInvocation>
+ getCompilerInvocation(const ParseInputs &Inputs) {
+ std::vector<std::string> CC1Args;
+ return buildCompilerInvocation(Inputs, DiagConsumer, &CC1Args);
+ }
+
+ SmallString<256> TestDir;
+ // Noticed MockFS but its member variable 'OverlayRealFileSystemForModules'
+ // implies that it will better to use RealThreadsafeFS directly.
+ RealThreadsafeFS TFS;
+
+ DiagnosticConsumer DiagConsumer;
+};
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
index 500b72b9b327a0..800d8c5450cb05 100644
--- a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
@@ -380,8 +380,10 @@ TEST(ParsedASTTest, PatchesAdditionalIncludes) {
MockFS FS;
auto Inputs = TU.inputs(FS);
auto CI = buildCompilerInvocation(Inputs, Diags);
+ MockCompilationDatabase CDB;
auto EmptyPreamble =
- buildPreamble(testPath("foo.cpp"), *CI, Inputs, true, nullptr);
+ buildPreamble(testPath("foo.cpp"), *CI, Inputs, true,
+ /*RequiredModuleBuilder=*/nullptr, nullptr);
ASSERT_TRUE(EmptyPreamble);
EXPECT_THAT(EmptyPreamble->Includes.MainFileIncludes, IsEmpty());
@@ -422,8 +424,10 @@ TEST(ParsedASTTest, PatchesDeletedIncludes) {
MockFS FS;
auto Inputs = TU.inputs(FS);
auto CI = buildCompilerInvocation(Inputs, Diags);
+ MockCompilationDatabase CDB;
auto BaselinePreamble =
- buildPreamble(testPath("foo.cpp"), *CI, Inputs, true, nullptr);
+ buildPreamble(testPath("foo.cpp"), *CI, Inputs, true,
+ /*RequiredModuleBuilder=*/nullptr, nullptr);
ASSERT_TRUE(BaselinePreamble);
EXPECT_THAT(BaselinePreamble->Includes.MainFileIncludes,
ElementsAre(testing::Field(&Inclusion::Written, "<foo.h>")));
diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
index 6420516e785576..149ca7819947b3 100644
--- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp
+++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
@@ -192,8 +192,10 @@ TEST(PreamblePatchTest, PatchesPreambleIncludes) {
TU.AdditionalFiles["b.h"] = "";
TU.AdditionalFiles["c.h"] = "";
auto PI = TU.inputs(FS);
- auto BaselinePreamble = buildPreamble(
- TU.Filename, *buildCompilerInvocation(PI, Diags), PI, true, nullptr);
+ MockCompilationDatabase CDB;
+ auto BaselinePreamble =
+ buildPreamble(TU.Filename, *buildCompilerInvocation(PI, Diags), PI, true,
+ /*RequiredModuleBuilder=*/nullptr, nullptr);
// We drop c.h from modified and add a new header. Since the latter is patched
// we should only get a.h in preamble includes. d.h shouldn't be part of the
// preamble, as it's coming from a disabled region.
diff --git a/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp b/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp
new file mode 100644
index 00000000000000..9107d0e2b85a5e
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp
@@ -0,0 +1,224 @@
+//===--------------- PrerequisiteModulesTests.cpp -------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+/// FIXME: Skip testing on windows temporarily due to the different escaping
+/// code mode.
+#ifndef _WIN32
+
+#include "PrerequisiteModules.h"
+#include "ModulesBuilder.h"
+#include "ModulesTestSetup.h"
+
+using namespace clang;
+using namespace clang::clangd;
+
+namespace {
+class PrerequisiteModulesTests : public ModuleTestSetup {};
+
+TEST_F(PrerequisiteModulesTests, PrerequisiteModulesTest) {
+ addFile("build/compile_commands.json", R"cpp(
+[
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/M.cppm -fmodule-output=__DIR__/M.pcm -c -o __DIR__/M.o",
+ "file": "__DIR__/M.cppm",
+ "output": "__DIR__/M.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/N.cppm -fmodule-file=M=__DIR__/M.pcm -fmodule-file=N:Part=__DIR__/N-partition.pcm -fprebuilt-module-path=__DIR__ -fmodule-output=__DIR__/N.pcm -c -o __DIR__/N.o",
+ "file": "__DIR__/N.cppm",
+ "output": "__DIR__/N.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/N-part.cppm -fmodule-output=__DIR__/N-partition.pcm -c -o __DIR__/N-part.o",
+ "file": "__DIR__/N-part.cppm",
+ "output": "__DIR__/N-part.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/L.cppm -fmodule-output=__DIR__/L.pcm -c -o __DIR__/L.o",
+ "file": "__DIR__/L.cppm",
+ "output": "__DIR__/L.o"
+},
+{
+ "directory": "__DIR__",
+ "command": "clang++ -std=c++20 __DIR__/NonModular.cpp -c -o __DIR__/NonModular.o",
+ "file": "__DIR__/NonModular.cpp",
+ "output": "__DIR__/NonModular.o"
+}
+]
+ )cpp");
+
+ addFile("foo.h", R"cpp(
+inline void foo() {}
+ )cpp");
+
+ addFile("M.cppm", R"cpp(
+module;
+#include "foo.h"
+export module M;
+ )cpp");
+
+ addFile("N.cppm", R"cpp(
+export module N;
+import :Part;
+import M;
+ )cpp");
+
+ addFile("N-part.cppm", R"cpp(
+// Different name with filename intentionally.
+export module N:Part;
+ )cpp");
+
+ addFile("bar.h", R"cpp(
+inline void bar() {}
+ )cpp");
+
+ addFile("L.cppm", R"cpp(
+module;
+#include "bar.h"
+export module L;
+ )cpp");
+
+ addFile("NonModular.cpp", R"cpp(
+#include "bar.h"
+#include "foo.h"
+void use() {
+ foo();
+ bar();
+}
+ )cpp");
+
+ std::unique_ptr<GlobalCompilationDatabase> CDB =
+ getGlobalCompilationDatabase();
+ EXPECT_TRUE(CDB);
+ ModulesBuilder Builder(*CDB.get());
+
+ // NonModular.cpp is not related to modules. So nothing should be built.
+ auto NonModularInfo =
+ Builder.buildPrerequisiteModulesFor(getFullPath("NonModular.cpp"), &TFS);
+ EXPECT_FALSE(NonModularInfo);
+
+ auto MInfo = Builder.buildPrerequisiteModulesFor(getFullPath("M.cppm"), &TFS);
+ // buildPrerequisiteModulesFor won't built the module itself.
+ EXPECT_FALSE(MInfo);
+
+ // Module N shouldn't be able to be built.
+ auto NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), &TFS);
+ EXPECT_TRUE(NInfo);
+ EXPECT_TRUE(NInfo->isModuleUnitBuilt("M"));
+ EXPECT_TRUE(NInfo->isModuleUnitBuilt("N:Part"));
+
+ ParseInputs NInput = getInputs("N.cppm", *CDB);
+ std::vector<std::string> CC1Args;
+ std::unique_ptr<CompilerInvocation> Invocation =
+ getCompilerInvocation(NInput);
+ // Test that `PrerequisiteModules::canReuse` works basically.
+ EXPECT_TRUE(NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ // Test that we can still reuse the NInfo after we touch a unrelated file.
+ {
+ addFile("L.cppm", R"cpp(
+module;
+#include "bar.h"
+export module L;
+export int ll = 43;
+ )cpp");
+ EXPECT_TRUE(NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ addFile("bar.h", R"cpp(
+inline void bar() {}
+inline void bar(int) {}
+ )cpp");
+ EXPECT_TRUE(NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+ }
+
+ // Test that we can't reuse the NInfo after we touch a related file.
+ {
+ addFile("M.cppm", R"cpp(
+module;
+#include "foo.h"
+export module M;
+export int mm = 44;
+ )cpp");
+ EXPECT_FALSE(NInfo && NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), &TFS);
+ EXPECT_TRUE(NInfo && NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ addFile("foo.h", R"cpp(
+inline void foo() {}
+inline void foo(int) {}
+ )cpp");
+ EXPECT_FALSE(NInfo && NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), &TFS);
+ EXPECT_TRUE(NInfo && NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+ }
+
+ addFile("N-part.cppm", R"cpp(
+export module N:Part;
+// Intentioned to make it uncompilable.
+export int NPart = 4LIdjwldijaw
+ )cpp");
+ EXPECT_FALSE(NInfo && NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+ NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), &TFS);
+ EXPECT_TRUE(NInfo);
+ // So NInfo should be unreusable even after rebuild.
+ EXPECT_FALSE(NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ addFile("N-part.cppm", R"cpp(
+export module N:Part;
+export int NPart = 43;
+ )cpp");
+ EXPECT_TRUE(NInfo);
+ EXPECT_FALSE(NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+ NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), &TFS);
+ // So NInfo should be unreusable even after rebuild.
+ EXPECT_TRUE(NInfo && NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ // Test that if we changed the modification time of the file, the module files
+ // info is still reusable if its content doesn't change.
+ addFile("N-part.cppm", R"cpp(
+export module N:Part;
+export int NPart = 43;
+ )cpp");
+ EXPECT_TRUE(NInfo && NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ addFile("N.cppm", R"cpp(
+export module N;
+import :Part;
+import M;
+
+export int nn = 43;
+ )cpp");
+ // NInfo should be reusable after we change its content.
+ EXPECT_TRUE(NInfo && NInfo->canReuse(*Invocation, TFS.view(TestDir)));
+
+ {
+ // Check that
+ // `PrerequisiteModules::adjustHeaderSearchOptions(HeaderSearchOptions&)`
+ // can replace HeaderSearchOptions correctly.
+ ParseInputs NInput = getInputs("N.cppm", *CDB);
+ std::vector<std::string> CC1Args;
+ std::unique_ptr<CompilerInvocation> NInvocation =
+ getCompilerInvocation(NInput);
+ HeaderSearchOptions &HSOpts = NInvocation->getHeaderSearchOpts();
+ NInfo->adjustHeaderSearchOptions(HSOpts);
+
+ EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.count("M"));
+ EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.count("N:Part"));
+ }
+}
+
+} // namespace
+
+#endif
diff --git a/clang-tools-extra/clangd/unittests/TestTU.cpp b/clang-tools-extra/clangd/unittests/TestTU.cpp
index e65ae825b41677..0d852d916fd9dd 100644
--- a/clang-tools-extra/clangd/unittests/TestTU.cpp
+++ b/clang-tools-extra/clangd/unittests/TestTU.cpp
@@ -107,8 +107,11 @@ TestTU::preamble(PreambleParsedCallback PreambleCallback) const {
initializeModuleCache(*CI);
auto ModuleCacheDeleter = llvm::make_scope_exit(
std::bind(deleteModuleCache, CI->getHeaderSearchOpts().ModuleCachePath));
+ MockCompilationDatabase CDB;
return clang::clangd::buildPreamble(testPath(Filename), *CI, Inputs,
- /*StoreInMemory=*/true, PreambleCallback);
+ /*StoreInMemory=*/true,
+ /*RequiredModuleBuilder=*/nullptr,
+ PreambleCallback);
}
ParsedAST TestTU::build() const {
@@ -123,9 +126,12 @@ ParsedAST TestTU::build() const {
auto ModuleCacheDeleter = llvm::make_scope_exit(
std::bind(deleteModuleCache, CI->getHeaderSearchOpts().ModuleCachePath));
- auto Preamble = clang::clangd::buildPreamble(testPath(Filename), *CI, Inputs,
- /*StoreInMemory=*/true,
- /*PreambleCallback=*/nullptr);
+ MockCompilationDatabase CDB;
+ auto Preamble =
+ clang::clangd::buildPreamble(testPath(Filename), *CI, Inputs,
+ /*StoreInMemory=*/true,
+ /*RequiredModuleBuilder=*/nullptr,
+ /*PreambleCallback=*/nullptr);
auto AST = ParsedAST::build(testPath(Filename), Inputs, std::move(CI),
Diags.take(), Preamble);
if (!AST) {
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index ecfb3aa9267f14..93578ffbaf89d5 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -48,6 +48,9 @@ Major New Features
Improvements to clangd
----------------------
+- Introduced exmperimental support for C++20 Modules. The experimental support can
+ be enabled by `-experimental-modules-support` option.
+
Inlay hints
^^^^^^^^^^^
More information about the cfe-commits
mailing list