[clang-tools-extra] [clangd] [C++20] [Modules] Introduce initial support for C++20 Modules (PR #66462)

kadir çetinkaya via cfe-commits cfe-commits at lists.llvm.org
Fri May 31 06:42:07 PDT 2024


================
@@ -0,0 +1,370 @@
+//===----------------- ModulesBuilder.cpp ------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ModulesBuilder.h"
+
+#include "Compiler.h"
+#include "support/Logger.h"
+
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/Frontend/FrontendActions.h"
+
+#include "clang/Serialization/ASTReader.h"
+
+namespace clang {
+namespace clangd {
+
+namespace {
+
+// Create a path to store module files. Generally it should be:
+//
+//   {TEMP_DIRS}/clangd/module_files/{PrefixDir}-%%-%%-%%-%%-%%-%%/.
+//
+// {TEMP_DIRS} is the temporary directory for the system, e.g., "/var/tmp"
+// or "C:/TEMP".
+//
+// '%%' means random value to make the generated path unique.
+//
+// \param MainFile is used to get the root of the project from global
+// compilation database. \param PrefixDir is used to get the user
+// defined prefix for module files. This is useful when we want to seperate
+// module files. e.g., we want to build module files for the same module unit
+// `a.cppm` with 2 different users `b.cpp` and `c.cpp` and we don't want the
+// module file for `b.cpp` be conflict with the module files for `c.cpp`. Then
+// we can put the 2 module files into different dirs like:
+//
+//   ${TEMP_DIRS}/clangd/module_files/b.cpp/a.pcm
+//   ${TEMP_DIRS}/clangd/module_files/c.cpp/a.pcm
+//
+// TODO: Move these module fils out of the temporary directory if the module
+// files are persistent.
+llvm::SmallString<256> getUniqueModuleFilesPath(PathRef MainFile,
+                                                llvm::StringRef PrefixDir) {
+  llvm::SmallString<256> ResultPattern;
+
+  llvm::sys::path::system_temp_directory(/*erasedOnReboot=*/true,
+                                         ResultPattern);
+
+  llvm::sys::path::append(ResultPattern, "clangd");
+  llvm::sys::path::append(ResultPattern, "module_files");
+
+  llvm::sys::path::append(ResultPattern, PrefixDir);
+
+  ResultPattern.append("-%%-%%-%%-%%-%%-%%");
+
+  llvm::SmallString<256> Result;
+  llvm::sys::fs::createUniquePath(ResultPattern, Result,
+                                  /*MakeAbsolute=*/false);
+
+  llvm::sys::fs::create_directories(Result);
+  return Result;
+}
+
+// Get the absolute path for the filename from the compile command.
+llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) {
+  llvm::SmallString<128> AbsolutePath;
+  if (llvm::sys::path::is_absolute(Cmd.Filename)) {
+    AbsolutePath = Cmd.Filename;
+  } else {
+    AbsolutePath = Cmd.Directory;
+    llvm::sys::path::append(AbsolutePath, Cmd.Filename);
+    llvm::sys::path::remove_dots(AbsolutePath, true);
+  }
+  return AbsolutePath;
+}
+
+// Get a unique module file path under \param ModuleFilesPrefix.
+std::string getModuleFilePath(llvm::StringRef ModuleName,
+                              PathRef ModuleFilesPrefix) {
+  llvm::SmallString<256> ModuleFilePattern(ModuleFilesPrefix);
+  auto [PrimaryModuleName, PartitionName] = ModuleName.split(':');
+  llvm::sys::path::append(ModuleFilePattern, PrimaryModuleName);
+  if (!PartitionName.empty()) {
+    ModuleFilePattern.append("-");
+    ModuleFilePattern.append(PartitionName);
+  }
+
+  ModuleFilePattern.append(".pcm");
+
+  llvm::SmallString<256> ModuleFilePath;
+  llvm::sys::fs::createUniquePath(ModuleFilePattern, ModuleFilePath,
+                                  /*MakeAbsolute=*/false);
+
+  return std::string(ModuleFilePath);
+}
+} // namespace
+
+// FailedPrerequisiteModules - stands for the PrerequisiteModules which has
+// errors happened during the building process.
+class FailedPrerequisiteModules : public PrerequisiteModules {
+public:
+  ~FailedPrerequisiteModules() override = default;
+
+  // We shouldn't adjust the compilation commands based on
+  // FailedPrerequisiteModules.
+  void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
+  }
+
+  // FailedPrerequisiteModules can never be reused.
+  bool
+  canReuse(const CompilerInvocation &CI,
+           llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override {
+    return false;
+  }
+
+  // No module unit got built in FailedPrerequisiteModules.
+  bool isModuleUnitBuilt(llvm::StringRef ModuleName) const override {
+    return false;
+  }
+};
+
+// StandalonePrerequisiteModules - stands for PrerequisiteModules for which all
+// the required modules are built successfully. All the module files
+// are owned by the StandalonePrerequisiteModules class.
+//
+// Any of the built module files won't be shared with other instances of the
+// class. So that we can avoid worrying thread safety.
+//
+// We don't need to worry about duplicated module names here since the standard
+// guarantees the module names should be unique to a program.
+class StandalonePrerequisiteModules : public PrerequisiteModules {
+public:
+  StandalonePrerequisiteModules() = default;
+
+  StandalonePrerequisiteModules(const StandalonePrerequisiteModules &) = delete;
+  StandalonePrerequisiteModules
+  operator=(const StandalonePrerequisiteModules &) = delete;
+  StandalonePrerequisiteModules(StandalonePrerequisiteModules &&) = delete;
+  StandalonePrerequisiteModules
+  operator=(StandalonePrerequisiteModules &&) = delete;
+
+  ~StandalonePrerequisiteModules() override = default;
+
+  void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
+    // Appending all built module files.
+    for (auto &RequiredModule : RequiredModules)
+      Options.PrebuiltModuleFiles.insert_or_assign(
+          RequiredModule.ModuleName, RequiredModule.ModuleFilePath);
+  }
+
+  bool canReuse(const CompilerInvocation &CI,
+                llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override;
+
+  bool isModuleUnitBuilt(llvm::StringRef ModuleName) const override {
+    return BuiltModuleNames.contains(ModuleName);
+  }
+
+  void addModuleFile(llvm::StringRef ModuleName,
+                     llvm::StringRef ModuleFilePath) {
+    RequiredModules.emplace_back(ModuleName, ModuleFilePath);
+    BuiltModuleNames.insert(ModuleName);
+  }
+
+private:
+  struct ModuleFile {
+    ModuleFile(llvm::StringRef ModuleName, PathRef ModuleFilePath)
+        : ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()) {}
+
+    ModuleFile() = delete;
+
+    ModuleFile(const ModuleFile &) = delete;
+    ModuleFile operator=(const ModuleFile &) = delete;
+
+    // The move constructor is needed for llvm::SmallVector.
+    ModuleFile(ModuleFile &&Other)
+        : ModuleName(std::move(Other.ModuleName)),
+          ModuleFilePath(std::move(Other.ModuleFilePath)) {}
+
+    ModuleFile &operator=(ModuleFile &&Other) = delete;
+
+    ~ModuleFile() {
+      if (!ModuleFilePath.empty())
+        llvm::sys::fs::remove(ModuleFilePath);
+    }
+
+    std::string ModuleName;
+    std::string ModuleFilePath;
+  };
+
+  llvm::SmallVector<ModuleFile, 8> RequiredModules;
+  // A helper class to speedup the query if a module is built.
+  llvm::StringSet<> BuiltModuleNames;
+};
+
+namespace {
+// Build a module file for module with `ModuleName`. Return false
+// when there are problem happens. Return true when the
+// module file exists or built successfully. The information of built
+// module file are stored in \param BuiltModuleFiles.
+bool buildModuleFile(llvm::StringRef ModuleName,
+                     const GlobalCompilationDatabase &CDB,
+                     const ThreadsafeFS *TFS, ProjectModules *MDB,
+                     PathRef ModuleFilesPrefix,
+                     StandalonePrerequisiteModules &BuiltModuleFiles) {
+  if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
+    return true;
+
+  PathRef ModuleUnitFileName = MDB->getSourceForModuleName(ModuleName);
+  // It is possible that we're meeting third party modules (modules whose
+  // source are not in the project. e.g, the std module may be a third-party
+  // module for most projects) or something wrong with the implementation of
+  // ProjectModules.
+  // FIXME: How should we treat third party modules here? If we want to ignore
+  // third party modules, we should return true instead of false here.
+  // Currently we simply bail out.
+  if (ModuleUnitFileName.empty()) {
+    elog("Failed to get the source for module name {0}. Maybe it is from third "
+         "party libraries or something goes wrong",
+         ModuleName);
+    return false;
+  }
+
+  for (auto &RequiredModuleName : MDB->getRequiredModules(ModuleUnitFileName)) {
+    // Return early if there are errors building the module file.
+    if (!buildModuleFile(RequiredModuleName, CDB, TFS, MDB, ModuleFilesPrefix,
+                         BuiltModuleFiles)) {
+      elog("Failed to build {0} since failed to build {1}", ModuleName,
+           RequiredModuleName);
+      return false;
+    }
+  }
+
+  auto Cmd = CDB.getCompileCommand(ModuleUnitFileName);
+  if (!Cmd) {
+    elog("Failed to get compile command for {0}", ModuleName);
+    return false;
+  }
+
+  Cmd->Output = getModuleFilePath(ModuleName, ModuleFilesPrefix);
+
+  ParseInputs Inputs;
+  Inputs.TFS = TFS;
+  Inputs.CompileCommand = std::move(*Cmd);
+
+  IgnoreDiagnostics IgnoreDiags;
+  auto CI = buildCompilerInvocation(Inputs, IgnoreDiags);
+  if (!CI) {
+    elog("Failed to build compiler invocation for {0}", ModuleName);
+    return false;
+  }
+
+  auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
+  auto AbsolutePath = getAbsolutePath(Inputs.CompileCommand);
+  auto Buf = FS->getBufferForFile(AbsolutePath);
+  if (!Buf) {
+    elog("Failed to create buffer for {0}", AbsolutePath);
+    return false;
+  }
+
+  // Hash the contents of input files and store the hash value to the BMI files.
+  // So that we can check if the files are still valid when we want to reuse the
+  // BMI files.
+  CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true;
+
+  BuiltModuleFiles.adjustHeaderSearchOptions(CI->getHeaderSearchOpts());
+
+  CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output;
+  auto Clang =
+      prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr,
+                              std::move(*Buf), std::move(FS), IgnoreDiags);
+  if (!Clang) {
+    elog("Failed to prepare compiler instance for {0}", AbsolutePath);
+    return false;
+  }
+
+  GenerateModuleInterfaceAction Action;
+  Clang->ExecuteAction(Action);
+
+  if (Clang->getDiagnostics().hasErrorOccurred()) {
+    elog("Compilation for {0} failed", AbsolutePath);
+    return false;
+  }
+
+  BuiltModuleFiles.addModuleFile(ModuleName, Inputs.CompileCommand.Output);
+  return true;
+}
+} // namespace
+
+std::unique_ptr<PrerequisiteModules>
+ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
+                                            const ThreadsafeFS *TFS) const {
+  std::unique_ptr<ProjectModules> MDB = CDB.getProjectModules(File);
+  if (!MDB)
+    return {};
+
+  std::vector<std::string> RequiredModuleNames = MDB->getRequiredModules(File);
+  if (RequiredModuleNames.empty())
+    return {};
+
+  llvm::SmallString<128> Prefix = llvm::sys::path::filename(File);
+  // There might be multiple files with the same name in a project. So appending
+  // the hash value of the full path to make sure they won't conflict.
+  Prefix += std::to_string(llvm::hash_value(File));
----------------
kadircet wrote:

can we move these into `getUniqueModuleFilesPath` and only pass `File`?

https://github.com/llvm/llvm-project/pull/66462


More information about the cfe-commits mailing list