[clang-tools-extra] [clangd] [Modules] Support Reusable Modules Builder (PR #106683)

[Chuanqi Xu via cfe-commits]

================
@@ -316,36 +295,187 @@ llvm::Error buildModuleFile(llvm::StringRef ModuleName,
   if (Clang->getDiagnostics().hasErrorOccurred())
     return llvm::createStringError("Compilation failed");
 
-  BuiltModuleFiles.addModuleFile(ModuleName, Inputs.CompileCommand.Output);
-  return llvm::Error::success();
+  return ModuleFile{ModuleName, Inputs.CompileCommand.Output};
 }
+
+bool ReusablePrerequisiteModules::canReuse(
+    const CompilerInvocation &CI,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) const {
+  if (RequiredModules.empty())
+    return true;
+
+  SmallVector<StringRef> BMIPaths;
+  for (auto &MF : RequiredModules)
+    BMIPaths.push_back(MF->getModuleFilePath());
+  return IsModuleFilesUpToDate(BMIPaths, *this, VFS);
+}
+
+class ModuleFileCache {
+public:
+  ModuleFileCache(const GlobalCompilationDatabase &CDB) : CDB(CDB) {}
+  const GlobalCompilationDatabase &getCDB() const { return CDB; }
+
+  std::shared_ptr<const ModuleFile> getModule(StringRef ModuleName);
+
+  void add(StringRef ModuleName, std::shared_ptr<const ModuleFile> ModuleFile) {
+    std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
+
+    ModuleFiles.insert_or_assign(ModuleName, ModuleFile);
+  }
+
+  void remove(StringRef ModuleName);
+
+private:
+  const GlobalCompilationDatabase &CDB;
+
+  llvm::StringMap<std::weak_ptr<const ModuleFile>> ModuleFiles;
+  // Mutex to guard accesses to ModuleFiles.
+  std::mutex ModuleFilesMutex;
+};
+
+std::shared_ptr<const ModuleFile>
+ModuleFileCache::getModule(StringRef ModuleName) {
+  std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
+
+  auto Iter = ModuleFiles.find(ModuleName);
+  if (Iter == ModuleFiles.end())
+    return nullptr;
+
+  if (Iter->second.expired()) {
+    ModuleFiles.erase(Iter);
+    return nullptr;
+  }
+
+  return Iter->second.lock();
+}
+
+void ModuleFileCache::remove(StringRef ModuleName) {
+  std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
+
+  auto Iter = ModuleFiles.find(ModuleName);
+  if (Iter == ModuleFiles.end())
+    return;
+
+  ModuleFiles.erase(Iter);
+}
+
+/// Collect the directly and indirectly required module names for \param
+/// ModuleName in topological order. The \param ModuleName is guaranteed to
+/// be the last element in \param ModuleNames.
+llvm::SmallVector<StringRef> getAllRequiredModules(ProjectModules &MDB,
+                                                   StringRef ModuleName) {
+  llvm::SmallVector<StringRef> ModuleNames;
+  llvm::StringSet<> ModuleNamesSet;
+
+  auto traversal = [&](StringRef ModuleName, auto recursionHelper) -> void {
+    ModuleNamesSet.insert(ModuleName);
+
+    for (StringRef RequiredModuleName :
+         MDB.getRequiredModules(MDB.getSourceForModuleName(ModuleName)))
+      if (ModuleNamesSet.insert(RequiredModuleName).second)
+        recursionHelper(RequiredModuleName, recursionHelper);
+
+    ModuleNames.push_back(ModuleName);
+  };
+  traversal(ModuleName, traversal);
+
+  return ModuleNames;
+}
+
 } // namespace
 
+class ModulesBuilder::ModulesBuilderImpl {
+public:
+  ModulesBuilderImpl(const GlobalCompilationDatabase &CDB) : Cache(CDB) {}
+
+  const GlobalCompilationDatabase &getCDB() const { return Cache.getCDB(); }
+
+  llvm::Error
+  getOrBuildModuleFile(StringRef ModuleName, const ThreadsafeFS &TFS,
+                       ProjectModules &MDB,
+                       ReusablePrerequisiteModules &BuiltModuleFiles);
+
+private:
+  ModuleFileCache Cache;
+};
+
+llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile(
+    StringRef ModuleName, const ThreadsafeFS &TFS, ProjectModules &MDB,
+    ReusablePrerequisiteModules &BuiltModuleFiles) {
+  if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
+    return llvm::Error::success();
+
+  PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName);
+  /// It is possible that we're meeting third party modules (modules whose
+  /// source are not in the project. e.g, the std module may be a third-party
+  /// module for most project) or something wrong with the implementation of
+  /// ProjectModules.
+  /// FIXME: How should we treat third party modules here? If we want to ignore
+  /// third party modules, we should return true instead of false here.
+  /// Currently we simply bail out.
+  if (ModuleUnitFileName.empty())
+    return llvm::createStringError(
+        llvm::formatv("Don't get the module unit for module {0}", ModuleName));
+
+  // Get Required modules in topological order.
+  auto ReqModuleNames = getAllRequiredModules(MDB, ModuleName);
+  for (llvm::StringRef ReqModuleName : ReqModuleNames) {
+    if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
+      continue;
+
+    if (auto Cached = Cache.getModule(ReqModuleName)) {
+      if (IsModuleFileUpToDate(Cached->getModuleFilePath(), BuiltModuleFiles,
+                               TFS.view(std::nullopt))) {
+        log("Reusing module {0} from {1}", ModuleName,
+            Cached->getModuleFilePath());
+        BuiltModuleFiles.addModuleFile(std::move(Cached));
+        continue;
+      }
+      Cache.remove(ReqModuleName);
+    }
+
+    llvm::SmallString<256> ModuleFilesPrefix =
+        getUniqueModuleFilesPath(ModuleUnitFileName);
+
+    llvm::Expected<ModuleFile> MF =
+        buildModuleFile(ModuleName, ModuleUnitFileName, getCDB(), TFS,
+                        ModuleFilesPrefix, BuiltModuleFiles);
+    if (llvm::Error Err = MF.takeError())
+      return Err;
+
+    log("Built module {0} to {1}", ModuleName, MF->getModuleFilePath());
+    auto BuiltModuleFile = std::make_shared<const ModuleFile>(std::move(*MF));
+    Cache.add(ModuleName, BuiltModuleFile);
+    BuiltModuleFiles.addModuleFile(std::move(BuiltModuleFile));
+  }
+
+  return llvm::Error::success();
+}
+
 std::unique_ptr<PrerequisiteModules>
 ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
-                                            const ThreadsafeFS &TFS) const {
-  std::unique_ptr<ProjectModules> MDB = CDB.getProjectModules(File);
+                                            const ThreadsafeFS &TFS) {
+  std::unique_ptr<ProjectModules> MDB = Impl->getCDB().getProjectModules(File);
   if (!MDB) {
     elog("Failed to get Project Modules information for {0}", File);
     return std::make_unique<FailedPrerequisiteModules>();
   }
 
   std::vector<std::string> RequiredModuleNames = MDB->getRequiredModules(File);
   if (RequiredModuleNames.empty())
-    return std::make_unique<StandalonePrerequisiteModules>();
+    return std::make_unique<ReusablePrerequisiteModules>();
 
   llvm::SmallString<256> ModuleFilesPrefix = getUniqueModuleFilesPath(File);
 
   log("Trying to build required modules for {0} in {1}", File,
       ModuleFilesPrefix);
----------------
ChuanqiXu9 wrote:

Done

https://github.com/llvm/llvm-project/pull/106683