[clang-tools-extra] [clangd] [Modules] Support Reusable Modules Builder (PR #106683)

[Chuanqi Xu via cfe-commits]

================
@@ -316,36 +309,221 @@ llvm::Error buildModuleFile(llvm::StringRef ModuleName,
   if (Clang->getDiagnostics().hasErrorOccurred())
     return llvm::createStringError("Compilation failed");
 
-  BuiltModuleFiles.addModuleFile(ModuleName, Inputs.CompileCommand.Output);
-  return llvm::Error::success();
+  return ModuleFile{ModuleName, Inputs.CompileCommand.Output, BuiltModuleFiles};
+}
+
+bool ReusablePrerequisiteModules::canReuse(
+    const CompilerInvocation &CI,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) const {
+  if (RequiredModules.empty())
+    return true;
+
+  SmallVector<StringRef> BMIPaths;
+  for (auto &MF : RequiredModules)
+    BMIPaths.push_back(MF->getModuleFilePath());
+  return IsModuleFilesUpToDate(BMIPaths, *this, VFS);
+}
+
+class ModuleFileCache {
+public:
+  ModuleFileCache(const GlobalCompilationDatabase &CDB) : CDB(CDB) {}
+
+  llvm::Error
+  getOrBuildModuleFile(StringRef ModuleName, const ThreadsafeFS &TFS,
+                       ProjectModules &MDB,
+                       ReusablePrerequisiteModules &RequiredModules);
+  const GlobalCompilationDatabase &getCDB() const { return CDB; }
+
+  std::shared_ptr<const ModuleFile>
+  getValidModuleFile(StringRef ModuleName, ProjectModules &MDB,
+                     const ThreadsafeFS &TFS,
+                     PrerequisiteModules &BuiltModuleFiles);
+
+  void add(StringRef ModuleName, std::shared_ptr<const ModuleFile> ModuleFile) {
+    std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
+
+    ModuleFiles.insert_or_assign(ModuleName, ModuleFile);
+  }
+
+private:
+  const GlobalCompilationDatabase &CDB;
+
+  llvm::StringMap<std::weak_ptr<const ModuleFile>> ModuleFiles;
+  // Mutex to guard accesses to ModuleFiles.
+  std::mutex ModuleFilesMutex;
+};
+
+/// Collect the directly and indirectly required module names for \param
+/// ModuleName. The \param ModuleName is guaranteed to be the first element in
+/// \param ModuleNames.
+llvm::SmallVector<StringRef> getAllRequiredModules(ProjectModules &MDB,
+                                                   StringRef ModuleName) {
+  llvm::SmallVector<StringRef> ModuleNames;
+
+  std::queue<StringRef> Worklist;
+  llvm::StringSet<> ModuleNamesSet;
+  Worklist.push(ModuleName);
+
+  while (!Worklist.empty()) {
+    StringRef CurrentModule = Worklist.front();
+    Worklist.pop();
+
+    if (!ModuleNamesSet.insert(CurrentModule).second)
+      continue;
+
+    ModuleNames.push_back(CurrentModule);
+
+    for (StringRef RequiredModuleName :
+         MDB.getRequiredModules(MDB.getSourceForModuleName(CurrentModule)))
+      if (!ModuleNamesSet.contains(RequiredModuleName))
+        Worklist.push(RequiredModuleName);
+  }
+
+  return ModuleNames;
+}
+
+std::shared_ptr<const ModuleFile>
+ModuleFileCache::getValidModuleFile(StringRef ModuleName, ProjectModules &MDB,
+                                    const ThreadsafeFS &TFS,
+                                    PrerequisiteModules &BuiltModuleFiles) {
+  {
+    std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
+
+    auto Iter = ModuleFiles.find(ModuleName);
+    if (Iter == ModuleFiles.end())
+      return nullptr;
+
+    if (Iter->second.expired()) {
+      ModuleFiles.erase(Iter);
+      return nullptr;
+    }
+  }
+
+  llvm::SmallVector<StringRef> ModuleNames =
+      getAllRequiredModules(MDB, ModuleName);
+
+  llvm::SmallVector<std::shared_ptr<const ModuleFile>> RequiredModuleFiles;
+
+  {
+    std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
+
+    for (StringRef ModuleName : ModuleNames) {
+      auto Iter = ModuleFiles.find(ModuleName);
+      if (Iter == ModuleFiles.end())
+        return nullptr;
+
+      if (Iter->second.expired()) {
+        ModuleFiles.erase(Iter);
+        return nullptr;
+      }
+
+      RequiredModuleFiles.push_back(Iter->second.lock());
+    }
+  }
+
+  assert(!RequiredModuleFiles.empty());
+
+  if (llvm::any_of(RequiredModuleFiles,
+                   [&](std::shared_ptr<const ModuleFile> MF) {
+                     return !IsModuleFileUpToDate(MF->getModuleFilePath(),
+                                                  BuiltModuleFiles,
+                                                  TFS.view(std::nullopt));
+                   }))
+    return nullptr;
+
+  return RequiredModuleFiles[0];
 }
 } // namespace
 
+class ModulesBuilder::ModulesBuilderImpl {
+public:
+  ModulesBuilderImpl(const GlobalCompilationDatabase &CDB) : Cache(CDB) {}
+
+  const GlobalCompilationDatabase &getCDB() const { return Cache.getCDB(); }
+
+  llvm::Error
+  getOrBuildModuleFile(StringRef ModuleName, const ThreadsafeFS &TFS,
+                       ProjectModules &MDB,
+                       ReusablePrerequisiteModules &BuiltModuleFiles);
+
+private:
+  ModuleFileCache Cache;
+};
+
+llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile(
+    StringRef ModuleName, const ThreadsafeFS &TFS, ProjectModules &MDB,
+    ReusablePrerequisiteModules &BuiltModuleFiles) {
+  if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
+    return llvm::Error::success();
+
+  PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName);
+  /// It is possible that we're meeting third party modules (modules whose
+  /// source are not in the project. e.g, the std module may be a third-party
+  /// module for most project) or something wrong with the implementation of
+  /// ProjectModules.
+  /// FIXME: How should we treat third party modules here? If we want to ignore
+  /// third party modules, we should return true instead of false here.
+  /// Currently we simply bail out.
+  if (ModuleUnitFileName.empty())
+    return llvm::createStringError(
+        llvm::formatv("Don't get the module unit for module {0}", ModuleName));
+
+  for (auto &RequiredModuleName : MDB.getRequiredModules(ModuleUnitFileName))
+    // Return early if there are errors building the module file.
+    if (!getOrBuildModuleFile(RequiredModuleName, TFS, MDB, BuiltModuleFiles))
+      return llvm::createStringError(
+          llvm::formatv("Failed to build module {0}", RequiredModuleName));
+
+  if (std::shared_ptr<const ModuleFile> Cached =
+          Cache.getValidModuleFile(ModuleName, MDB, TFS, BuiltModuleFiles)) {
----------------
ChuanqiXu9 wrote:

Excellent! I accept the suggestion to avoid the signature. Since we may need to share `ReusablePrerequisiteModules` for different modules. See `ModulesBuilder::buildPrerequisiteModulesFor`.

https://github.com/llvm/llvm-project/pull/106683