[clang-tools-extra] [clangd] [C++20] [Modules] Introduce initial support for C++20 Modules (PR #66462)

[Chuanqi Xu via cfe-commits]

================
@@ -0,0 +1,282 @@
+//===----------------- ModuleFilesInfo.cpp -----------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ModuleFilesInfo.h"
+#include "support/Logger.h"
+
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Serialization/ASTReader.h"
+
+namespace clang {
+namespace clangd {
+
+namespace {
+llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) {
+  llvm::SmallString<128> AbsolutePath;
+  if (llvm::sys::path::is_absolute(Cmd.Filename)) {
+    AbsolutePath = Cmd.Filename;
+  } else {
+    AbsolutePath = Cmd.Directory;
+    llvm::sys::path::append(AbsolutePath, Cmd.Filename);
+    llvm::sys::path::remove_dots(AbsolutePath, true);
+  }
+  return AbsolutePath;
+}
+} // namespace
+
+ModuleFilesInfo::ModuleFilesInfo(PathRef MainFile,
+                                 const GlobalCompilationDatabase &CDB) {
+  std::optional<ProjectInfo> PI = CDB.getProjectInfo(MainFile);
+  if (!PI)
+    return;
+
+  llvm::SmallString<128> Result(PI->SourceRoot);
+  llvm::sys::path::append(Result, ".cache");
+  llvm::sys::path::append(Result, "clangd");
+  llvm::sys::path::append(Result, "module_files");
+  llvm::sys::fs::create_directories(Result, /*IgnoreExisting=*/true);
+
+  llvm::sys::path::append(Result, llvm::sys::path::filename(MainFile));
+  llvm::sys::fs::createUniqueDirectory(Result, UniqueModuleFilesPathPrefix);
+
+  log("Initialized module files to {0}", UniqueModuleFilesPathPrefix.str());
+}
+
+ModuleFilesInfo::~ModuleFilesInfo() {
+  DependentModuleNames.clear();
+  Successed = false;
+
+  if (UniqueModuleFilesPathPrefix.empty())
+    return;
+
+  llvm::sys::fs::remove_directories(UniqueModuleFilesPathPrefix);
+  UniqueModuleFilesPathPrefix.clear();
+}
+
+llvm::SmallString<256>
+ModuleFilesInfo::getModuleFilePath(StringRef ModuleName) const {
+  llvm::SmallString<256> ModuleFilePath;
+
+  ModuleFilePath = UniqueModuleFilesPathPrefix;
+  auto [PrimaryModuleName, PartitionName] = ModuleName.split(':');
+  llvm::sys::path::append(ModuleFilePath, PrimaryModuleName);
+  if (!PartitionName.empty()) {
+    ModuleFilePath.append("-");
+    ModuleFilePath.append(PartitionName);
+  }
+  ModuleFilePath.append(".pcm");
+
+  return ModuleFilePath;
+}
+
+bool ModuleFilesInfo::IsModuleUnitBuilt(StringRef ModuleName) const {
+  if (!DependentModuleNames.count(ModuleName))
+    return false;
+
+  auto BMIPath = getModuleFilePath(ModuleName);
+  if (llvm::sys::fs::exists(BMIPath))
+    return true;
+
+  Successed = false;
+
+  DependentModuleNames.erase(ModuleName);
+  return false;
+}
+
+void ModuleFilesInfo::ReplaceHeaderSearchOptions(
+    HeaderSearchOptions &Options) const {
+  if (!IsInited())
+    return;
+
+  Options.PrebuiltModulePaths.insert(Options.PrebuiltModulePaths.begin(),
+                                     UniqueModuleFilesPathPrefix.str().str());
+
+  for (auto Iter = Options.PrebuiltModuleFiles.begin();
+       Iter != Options.PrebuiltModuleFiles.end();) {
+    if (IsModuleUnitBuilt(Iter->first)) {
+      Iter = Options.PrebuiltModuleFiles.erase(Iter);
+      continue;
+    }
+
+    Iter++;
+  }
+}
+
+void ModuleFilesInfo::ReplaceCompileCommands(
+    tooling::CompileCommand &Cmd) const {
+  if (!IsInited())
+    return;
+
+  std::vector<std::string> CommandLine(std::move(Cmd.CommandLine));
+
+  Cmd.CommandLine.emplace_back(CommandLine[0]);
+  Cmd.CommandLine.emplace_back(
+      llvm::Twine("-fprebuilt-module-path=" + UniqueModuleFilesPathPrefix)
+          .str());
+
+  for (std::size_t I = 1; I < CommandLine.size(); I++) {
+    const std::string &Arg = CommandLine[I];
+    const auto &[LHS, RHS] = StringRef(Arg).split("=");
+
+    // Remove original `-fmodule-file=<module-name>=<module-path>` form if it
+    // already built.
+    if (LHS == "-fmodule-file" && RHS.contains("=")) {
+      const auto &[ModuleName, _] = RHS.split("=");
+      if (IsModuleUnitBuilt(ModuleName))
+        continue;
+    }
+
+    Cmd.CommandLine.emplace_back(Arg);
+  }
+}
+
+void ModuleFilesInfo::ReplaceCompileCommands(tooling::CompileCommand &Cmd,
+                                             StringRef OutputModuleName) const {
+  if (!IsInited())
+    return;
+
+  ReplaceCompileCommands(Cmd);
+
+  Cmd.Output = getModuleFilePath(OutputModuleName).str().str();
+}
+
+bool ModuleFilesInfo::buildModuleFile(PathRef ModuleUnitFileName,
+                                      ModuleDependencyScanner &Scanner) {
+  if (ModuleUnitFileName.empty())
+    return false;
+
+  for (auto &ModuleName : Scanner.getRequiredModules(ModuleUnitFileName)) {
+    // Return early if there are errors building the module file.
+    if (!IsModuleUnitBuilt(ModuleName) &&
+        !buildModuleFile(Scanner.getSourceForModuleName(ModuleName), Scanner)) {
+      log("Failed to build module {0}", ModuleName);
+      return false;
+    }
+  }
+
+  auto Cmd =
+      Scanner.getCompilationDatabase().getCompileCommand(ModuleUnitFileName);
+  if (!Cmd)
+    return false;
+
+  ReplaceCompileCommands(*Cmd, Scanner.getModuleName(ModuleUnitFileName));
+
+  ParseInputs Inputs;
+  Inputs.TFS = Scanner.getThreadsafeFS();
+  Inputs.CompileCommand = std::move(*Cmd);
+
+  IgnoreDiagnostics IgnoreDiags;
+  auto CI = buildCompilerInvocation(Inputs, IgnoreDiags);
+  if (!CI)
+    return false;
+
+  auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
+  auto AbsolutePath = getAbsolutePath(Inputs.CompileCommand);
+  auto Buf = FS->getBufferForFile(AbsolutePath);
+  if (!Buf)
+    return false;
+
+  // Hash the contents of input files and store the hash value to the BMI files.
+  // So that we can check if the files are still valid when we want to reuse the
+  // BMI files.
+  CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true;
+
+  CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output;
+  auto Clang =
+      prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr,
+                              std::move(*Buf), std::move(FS), IgnoreDiags);
+  if (!Clang)
+    return false;
+
+  GenerateModuleInterfaceAction Action;
+  Clang->ExecuteAction(Action);
+
+  if (Clang->getDiagnostics().hasErrorOccurred())
+    return false;
+
+  DependentModuleNames.insert(Scanner.getModuleName(ModuleUnitFileName));
+
+  return true;
+}
+
+ModuleFilesInfo
+ModuleFilesInfo::buildModuleFilesInfoFor(PathRef File, const ThreadsafeFS *TFS,
+                                         const GlobalCompilationDatabase &CDB) {
+  ModuleDependencyScanner Scanner(CDB, TFS);
+
+  std::optional<tooling::dependencies::P1689Rule> ScanningResult =
+      Scanner.scan(File);
+  if (!ScanningResult)
+    return {};
+
+  ModuleFilesInfo ModulesInfo(File, CDB);
+
+  Scanner.globalScan(File);
+
+  for (auto &Info : ScanningResult->Requires)
+    // Return early if there is any error.
+    if (!ModulesInfo.buildModuleFile(
+            Scanner.getSourceForModuleName(Info.ModuleName), Scanner)) {
+      log("Failed to build module {0}", Info.ModuleName);
+      return ModulesInfo;
+    }
+
+  ModulesInfo.Successed = true;
+  return ModulesInfo;
+}
+
+bool ModuleFilesInfo::CanReuse(
+    const CompilerInvocation &CI,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) const {
+  // Try to avoid expensive check as much as possible.
+  if (!IsInited())
+    return true;
+
+  if (!Successed)
----------------
ChuanqiXu9 wrote:

Done by removing `Successed`.

https://github.com/llvm/llvm-project/pull/66462