[clang-tools-extra] [clangd] [C++20] [Modules] Introduce initial support for C++20 Modules (PR #66462)
via cfe-commits
cfe-commits at lists.llvm.org
Thu Sep 14 23:26:52 PDT 2023
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang-modules
<details>
<summary>Changes</summary>
(to be edited)
(not ready for review yet)
--
Patch is 66.97 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/66462.diff
28 Files Affected:
- (modified) clang-tools-extra/clangd/CMakeLists.txt (+3)
- (modified) clang-tools-extra/clangd/ClangdServer.cpp (+1)
- (modified) clang-tools-extra/clangd/ClangdServer.h (+3)
- (modified) clang-tools-extra/clangd/GlobalCompilationDatabase.cpp (+21)
- (modified) clang-tools-extra/clangd/GlobalCompilationDatabase.h (+6)
- (added) clang-tools-extra/clangd/ModuleDependencyScanner.cpp (+86)
- (added) clang-tools-extra/clangd/ModuleDependencyScanner.h (+78)
- (added) clang-tools-extra/clangd/ModuleFilesInfo.cpp (+282)
- (added) clang-tools-extra/clangd/ModuleFilesInfo.h (+118)
- (modified) clang-tools-extra/clangd/ParsedAST.cpp (+8)
- (modified) clang-tools-extra/clangd/Preamble.cpp (+17-6)
- (modified) clang-tools-extra/clangd/Preamble.h (+7)
- (modified) clang-tools-extra/clangd/TUScheduler.cpp (+13)
- (modified) clang-tools-extra/clangd/TUScheduler.h (+3)
- (modified) clang-tools-extra/clangd/test/CMakeLists.txt (+1)
- (added) clang-tools-extra/clangd/test/modules.test (+79)
- (modified) clang-tools-extra/clangd/tool/Check.cpp (+6-2)
- (modified) clang-tools-extra/clangd/tool/ClangdMain.cpp (+8)
- (modified) clang-tools-extra/clangd/unittests/CMakeLists.txt (+2)
- (modified) clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp (+14-4)
- (modified) clang-tools-extra/clangd/unittests/FileIndexTests.cpp (+4-1)
- (added) clang-tools-extra/clangd/unittests/ModuleDependencyScannerTest.cpp (+173)
- (added) clang-tools-extra/clangd/unittests/ModuleFilesInfoTest.cpp (+223)
- (added) clang-tools-extra/clangd/unittests/ModulesTestSetup.h (+105)
- (modified) clang-tools-extra/clangd/unittests/ParsedASTTests.cpp (+6-2)
- (modified) clang-tools-extra/clangd/unittests/PreambleTests.cpp (+4-2)
- (modified) clang-tools-extra/clangd/unittests/TestTU.cpp (+10-4)
- (modified) clang-tools-extra/docs/ReleaseNotes.rst (+3)
<pre>
diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index 3911fb6c6c746a8..bcfb49551a02591 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -97,6 +97,8 @@ add_clang_library(clangDaemon
IncludeFixer.cpp
InlayHints.cpp
JSONTransport.cpp
+ ModuleDependencyScanner.cpp
+ ModuleFilesInfo.cpp
PathMapping.cpp
Protocol.cpp
Quality.cpp
@@ -161,6 +163,7 @@ clang_target_link_libraries(clangDaemon
clangAST
clangASTMatchers
clangBasic
+ clangDependencyScanning
clangDriver
clangFormat
clangFrontend
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index 13d788162817fb4..e4c85858b6882ae 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -202,6 +202,7 @@ ClangdServer::Options::operator TUScheduler::Options() const {
Opts.UpdateDebounce = UpdateDebounce;
Opts.ContextProvider = ContextProvider;
Opts.PreambleThrottler = PreambleThrottler;
+ Opts.ExperimentalModulesSupport = ExperimentalModulesSupport;
return Opts;
}
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index a416602251428b0..dc546b118cb8f5e 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -112,6 +112,9 @@ class ClangdServer {
/// This throttler controls which preambles may be built at a given time.
clangd::PreambleThrottler *PreambleThrottler = nullptr;
+ /// Enable experimental support for modules.
+ bool ExperimentalModulesSupport = false;
+
/// If true, ClangdServer builds a dynamic in-memory index for symbols in
/// opened files and uses the index to augment code completion results.
bool BuildDynamicSymbolIndex = false;
diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
index d1833759917a30f..bcc8f4f0dd9e5ac 100644
--- a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
+++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
@@ -729,6 +729,20 @@ DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const {
return Res->PI;
}
+std::vector<std::string>
+DirectoryBasedGlobalCompilationDatabase::getAllFilesInProjectOf(
+ PathRef File) const {
+ CDBLookupRequest Req;
+ Req.FileName = File;
+ Req.ShouldBroadcast = false;
+ Req.FreshTime = Req.FreshTimeMissing =
+ std::chrono::steady_clock::time_point::min();
+ auto Res = lookupCDB(Req);
+ if (!Res)
+ return {};
+ return Res->CDB->getAllFiles();
+}
+
OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base,
std::vector<std::string> FallbackFlags,
CommandMangler Mangler)
@@ -805,6 +819,13 @@ std::optional<ProjectInfo> DelegatingCDB::getProjectInfo(PathRef File) const {
return Base->getProjectInfo(File);
}
+std::vector<std::string>
+DelegatingCDB::getAllFilesInProjectOf(PathRef File) const {
+ if (!Base)
+ return {};
+ return Base->getAllFilesInProjectOf(File);
+}
+
tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const {
if (!Base)
return GlobalCompilationDatabase::getFallbackCommand(File);
diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.h b/clang-tools-extra/clangd/GlobalCompilationDatabase.h
index 2bf8c973c534c6f..eaeff8d627a0960 100644
--- a/clang-tools-extra/clangd/GlobalCompilationDatabase.h
+++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.h
@@ -45,6 +45,10 @@ class GlobalCompilationDatabase {
return std::nullopt;
}
+ virtual std::vector<std::string> getAllFilesInProjectOf(PathRef File) const {
+ return {};
+ }
+
/// Makes a guess at how to build a file.
/// The default implementation just runs clang on the file.
/// Clangd should treat the results as unreliable.
@@ -75,6 +79,7 @@ class DelegatingCDB : public GlobalCompilationDatabase {
getCompileCommand(PathRef File) const override;
std::optional<ProjectInfo> getProjectInfo(PathRef File) const override;
+ std::vector<std::string> getAllFilesInProjectOf(PathRef File) const override;
tooling::CompileCommand getFallbackCommand(PathRef File) const override;
@@ -121,6 +126,7 @@ class DirectoryBasedGlobalCompilationDatabase
/// Returns the path to first directory containing a compilation database in
/// \p File's parents.
std::optional<ProjectInfo> getProjectInfo(PathRef File) const override;
+ std::vector<std::string> getAllFilesInProjectOf(PathRef File) const override;
bool blockUntilIdle(Deadline Timeout) const override;
diff --git a/clang-tools-extra/clangd/ModuleDependencyScanner.cpp b/clang-tools-extra/clangd/ModuleDependencyScanner.cpp
new file mode 100644
index 000000000000000..d706d2eb2fc8d6e
--- /dev/null
+++ b/clang-tools-extra/clangd/ModuleDependencyScanner.cpp
@@ -0,0 +1,86 @@
+//===---------------- ModuleDependencyScanner.cpp ----------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ModuleDependencyScanner.h"
+
+namespace clang {
+namespace clangd {
+using P1689Rule = tooling::dependencies::P1689Rule;
+
+std::optional<P1689Rule> ModuleDependencyScanner::scan(PathRef FilePath) {
+ if (ScanningCache.count(FilePath))
+ return ScanningCache[FilePath];
+
+ std::optional<tooling::CompileCommand> Cmd = CDB.getCompileCommand(FilePath);
+
+ if (!Cmd)
+ return std::nullopt;
+
+ using namespace clang::tooling::dependencies;
+
+ llvm::SmallString<128> FilePathDir(FilePath);
+ llvm::sys::path::remove_filename(FilePathDir);
+ DependencyScanningTool ScanningTool(
+ Service,
+ TFS ? TFS->view(FilePathDir) : llvm::vfs::createPhysicalFileSystem());
+
+ llvm::Expected<P1689Rule> Result =
+ ScanningTool.getP1689ModuleDependencyFile(*Cmd, Cmd->Directory);
+
+ if (auto E = Result.takeError()) {
+ // Ignore any error.
+ llvm::consumeError(std::move(E));
+ return std::nullopt;
+ }
+
+ if (Result->Provides)
+ ModuleNameToSourceMapper[Result->Provides->ModuleName] = FilePath;
+
+ ScanningCache[FilePath] = *Result;
+ return *Result;
+}
+
+void ModuleDependencyScanner::globalScan(PathRef File) {
+ std::vector<std::string> AllFiles = CDB.getAllFilesInProjectOf(File);
+
+ for (auto &File : AllFiles)
+ scan(File);
+}
+
+PathRef ModuleDependencyScanner::getSourceForModuleName(StringRef ModuleName) const {
+ if (!ModuleNameToSourceMapper.count(ModuleName))
+ return {};
+
+ return ModuleNameToSourceMapper[ModuleName];
+}
+
+std::vector<std::string>
+ModuleDependencyScanner::getRequiredModules(PathRef File) const {
+ if (!ScanningCache.count(File))
+ return {};
+
+ const P1689Rule &CachedResult = ScanningCache[File];
+ std::vector<std::string> Result;
+
+ for (const auto &Info : CachedResult.Requires)
+ Result.push_back(Info.ModuleName);
+
+ return Result;
+}
+
+StringRef ModuleDependencyScanner::getModuleName(PathRef File) const {
+ if (!ScanningCache.count(File))
+ return {};
+
+ if (!ScanningCache[File].Provides)
+ return {};
+
+ return ScanningCache[File].Provides->ModuleName;
+}
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/ModuleDependencyScanner.h b/clang-tools-extra/clangd/ModuleDependencyScanner.h
new file mode 100644
index 000000000000000..1d6eb58fda59e20
--- /dev/null
+++ b/clang-tools-extra/clangd/ModuleDependencyScanner.h
@@ -0,0 +1,78 @@
+//===-------------- ModuleDependencyScanner.h --------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULEDEPENDENCYSCANNER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULEDEPENDENCYSCANNER_H
+
+#include "GlobalCompilationDatabase.h"
+#include "support/Path.h"
+#include "support/ThreadsafeFS.h"
+
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringMap.h"
+
+namespace clang {
+namespace clangd {
+
+/// A scanner to produce P1689 format for C++20 Modules.
+///
+/// The scanner can scan a single file with `scan(PathRef)` member function
+/// or scan the whole project with `globalScan(PathRef)` member function. See
+/// the comments of `globalScan` to see the details.
+class ModuleDependencyScanner {
+public:
+ ModuleDependencyScanner(const GlobalCompilationDatabase &CDB,
+ const ThreadsafeFS *TFS)
+ : CDB(CDB), TFS(TFS),
+ Service(tooling::dependencies::ScanningMode::CanonicalPreprocessing,
+ tooling::dependencies::ScanningOutputFormat::P1689) {}
+
+ /// Scanning the single file specified by \param FilePath.
+ std::optional<clang::tooling::dependencies::P1689Rule> scan(PathRef FilePath);
+
+ /// Scanning every source file in the current project to get the
+ /// <module-name> to <module-unit-source> map.
+ /// It looks unefficiency to scan the whole project especially for
+ /// every version of every file!
+ /// TODO: We should find a efficient method to get the <module-name>
+ /// to <module-unit-source> map. We can make it either by providing
+ /// a global module dependency scanner to monitor every file. Or we
+ /// can simply require the build systems (or even if the end users)
+ /// to provide the map.
+ void globalScan(PathRef File);
+
+ PathRef getSourceForModuleName(StringRef ModuleName) const;
+
+ /// Return the direct required modules. Indirect required modules are not
+ /// included.
+ std::vector<std::string> getRequiredModules(PathRef File) const;
+ StringRef getModuleName(PathRef File) const;
+
+ const ThreadsafeFS *getThreadsafeFS() const { return TFS; }
+
+ const GlobalCompilationDatabase &getCompilationDatabase() const { return CDB; }
+
+private:
+ const GlobalCompilationDatabase &CDB;
+ const ThreadsafeFS *TFS;
+
+ clang::tooling::dependencies::DependencyScanningService Service;
+
+ // Map source file to P1689 Result.
+ llvm::StringMap<clang::tooling::dependencies::P1689Rule> ScanningCache;
+ // Map module name to source file path.
+ llvm::StringMap<std::string> ModuleNameToSourceMapper;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/ModuleFilesInfo.cpp b/clang-tools-extra/clangd/ModuleFilesInfo.cpp
new file mode 100644
index 000000000000000..845ff01ca09dff3
--- /dev/null
+++ b/clang-tools-extra/clangd/ModuleFilesInfo.cpp
@@ -0,0 +1,282 @@
+//===----------------- ModuleFilesInfo.cpp -----------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ModuleFilesInfo.h"
+#include "support/Logger.h"
+
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Serialization/ASTReader.h"
+
+namespace clang {
+namespace clangd {
+
+namespace {
+llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) {
+ llvm::SmallString<128> AbsolutePath;
+ if (llvm::sys::path::is_absolute(Cmd.Filename)) {
+ AbsolutePath = Cmd.Filename;
+ } else {
+ AbsolutePath = Cmd.Directory;
+ llvm::sys::path::append(AbsolutePath, Cmd.Filename);
+ llvm::sys::path::remove_dots(AbsolutePath, true);
+ }
+ return AbsolutePath;
+}
+} // namespace
+
+ModuleFilesInfo::ModuleFilesInfo(PathRef MainFile,
+ const GlobalCompilationDatabase &CDB) {
+ std::optional<ProjectInfo> PI = CDB.getProjectInfo(MainFile);
+ if (!PI)
+ return;
+
+ llvm::SmallString<128> Result(PI->SourceRoot);
+ llvm::sys::path::append(Result, ".cache");
+ llvm::sys::path::append(Result, "clangd");
+ llvm::sys::path::append(Result, "module_files");
+ llvm::sys::fs::create_directories(Result, /*IgnoreExisting=*/true);
+
+ llvm::sys::path::append(Result, llvm::sys::path::filename(MainFile));
+ llvm::sys::fs::createUniqueDirectory(Result, UniqueModuleFilesPathPrefix);
+
+ log("Initialized module files to {0}", UniqueModuleFilesPathPrefix.str());
+}
+
+ModuleFilesInfo::~ModuleFilesInfo() {
+ DependentModuleNames.clear();
+ Successed = false;
+
+ if (UniqueModuleFilesPathPrefix.empty())
+ return;
+
+ llvm::sys::fs::remove_directories(UniqueModuleFilesPathPrefix);
+ UniqueModuleFilesPathPrefix.clear();
+}
+
+llvm::SmallString<256>
+ModuleFilesInfo::getModuleFilePath(StringRef ModuleName) const {
+ llvm::SmallString<256> ModuleFilePath;
+
+ ModuleFilePath = UniqueModuleFilesPathPrefix;
+ auto [PrimaryModuleName, PartitionName] = ModuleName.split(':');
+ llvm::sys::path::append(ModuleFilePath, PrimaryModuleName);
+ if (!PartitionName.empty()) {
+ ModuleFilePath.append("-");
+ ModuleFilePath.append(PartitionName);
+ }
+ ModuleFilePath.append(".pcm");
+
+ return ModuleFilePath;
+}
+
+bool ModuleFilesInfo::IsModuleUnitBuilt(StringRef ModuleName) const {
+ if (!DependentModuleNames.count(ModuleName))
+ return false;
+
+ auto BMIPath = getModuleFilePath(ModuleName);
+ if (llvm::sys::fs::exists(BMIPath))
+ return true;
+
+ Successed = false;
+
+ DependentModuleNames.erase(ModuleName);
+ return false;
+}
+
+void ModuleFilesInfo::ReplaceHeaderSearchOptions(
+ HeaderSearchOptions &Options) const {
+ if (!IsInited())
+ return;
+
+ Options.PrebuiltModulePaths.insert(Options.PrebuiltModulePaths.begin(),
+ UniqueModuleFilesPathPrefix.str().str());
+
+ for (auto Iter = Options.PrebuiltModuleFiles.begin();
+ Iter != Options.PrebuiltModuleFiles.end();) {
+ if (IsModuleUnitBuilt(Iter->first)) {
+ Iter = Options.PrebuiltModuleFiles.erase(Iter);
+ continue;
+ }
+
+ Iter++;
+ }
+}
+
+void ModuleFilesInfo::ReplaceCompileCommands(
+ tooling::CompileCommand &Cmd) const {
+ if (!IsInited())
+ return;
+
+ std::vector<std::string> CommandLine(std::move(Cmd.CommandLine));
+
+ Cmd.CommandLine.emplace_back(CommandLine[0]);
+ Cmd.CommandLine.emplace_back(
+ llvm::Twine("-fprebuilt-module-path=" + UniqueModuleFilesPathPrefix)
+ .str());
+
+ for (std::size_t I = 1; I < CommandLine.size(); I++) {
+ const std::string &Arg = CommandLine[I];
+ const auto &[LHS, RHS] = StringRef(Arg).split("=");
+
+ // Remove original `-fmodule-file=<module-name>=<module-path>` form if it
+ // already built.
+ if (LHS == "-fmodule-file" && RHS.contains("=")) {
+ const auto &[ModuleName, _] = RHS.split("=");
+ if (IsModuleUnitBuilt(ModuleName))
+ continue;
+ }
+
+ Cmd.CommandLine.emplace_back(Arg);
+ }
+}
+
+void ModuleFilesInfo::ReplaceCompileCommands(tooling::CompileCommand &Cmd,
+ StringRef OutputModuleName) const {
+ if (!IsInited())
+ return;
+
+ ReplaceCompileCommands(Cmd);
+
+ Cmd.Output = getModuleFilePath(OutputModuleName).str().str();
+}
+
+bool ModuleFilesInfo::buildModuleFile(PathRef ModuleUnitFileName,
+ ModuleDependencyScanner &Scanner) {
+ if (ModuleUnitFileName.empty())
+ return false;
+
+ for (auto &ModuleName : Scanner.getRequiredModules(ModuleUnitFileName)) {
+ // Return early if there are errors building the module file.
+ if (!IsModuleUnitBuilt(ModuleName) &&
+ !buildModuleFile(Scanner.getSourceForModuleName(ModuleName), Scanner)) {
+ log("Failed to build module {0}", ModuleName);
+ return false;
+ }
+ }
+
+ auto Cmd =
+ Scanner.getCompilationDatabase().getCompileCommand(ModuleUnitFileName);
+ if (!Cmd)
+ return false;
+
+ ReplaceCompileCommands(*Cmd, Scanner.getModuleName(ModuleUnitFileName));
+
+ ParseInputs Inputs;
+ Inputs.TFS = Scanner.getThreadsafeFS();
+ Inputs.CompileCommand = std::move(*Cmd);
+
+ IgnoreDiagnostics IgnoreDiags;
+ auto CI = buildCompilerInvocation(Inputs, IgnoreDiags);
+ if (!CI)
+ return false;
+
+ auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
+ auto AbsolutePath = getAbsolutePath(Inputs.CompileCommand);
+ auto Buf = FS->getBufferForFile(AbsolutePath);
+ if (!Buf)
+ return false;
+
+ // Hash the contents of input files and store the hash value to the BMI files.
+ // So that we can check if the files are still valid when we want to reuse the
+ // BMI files.
+ CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true;
+
+ CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output;
+ auto Clang =
+ prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr,
+ std::move(*Buf), std::move(FS), IgnoreDiags);
+ if (!Clang)
+ retu...
<truncated>
</pre>
</details>
https://github.com/llvm/llvm-project/pull/66462
More information about the cfe-commits
mailing list