[clang] [clang][deps] Parallelize module compilations (PR #180047)

[Jan Svoboda via cfe-commits]

================
@@ -546,6 +553,127 @@ dependencies::initializeScanInstanceDependencyCollector(
   return MDC;
 }
 
+struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
+  DependencyScanningService &Service;
+
+  SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service)
+      : Service(Service) {}
+
+  bool BeginSourceFileAction(CompilerInstance &CI) override;
+};
+
+/// The preprocessor callback that takes care of initiating an asynchronous
+/// module compilation if needed.
+struct AsyncModuleCompile : PPCallbacks {
+  CompilerInstance &CI;
+  DependencyScanningService &Service;
+
+  AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service)
+      : CI(CI), Service(Service) {}
+
+  void moduleLoadSkipped(Module *M) override {
+    M = M->getTopLevelModule();
+
+    HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
+    ModuleCache &ModCache = CI.getModuleCache();
+    std::string ModuleFileName = HS.getCachedModuleFileName(M);
+
+    uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName);
+    // Someone else already built/validated the PCM.
+    if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
+      return;
+
+    if (!CI.getASTReader())
+      CI.createASTReader();
+    SmallVector<ASTReader::ImportedModule, 0> Imported;
+    // Only calling ReadASTCore() to avoid the expensive eager deserialization
+    // of the clang::Module objects in ReadAST().
+    // FIXME: Consider doing this in the new thread depending on how expensive
+    // the read turns out to be.
+    switch (CI.getASTReader()->ReadASTCore(
+        ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(),
+        nullptr, Imported, {}, {}, {},
+        ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
+            ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
+    case ASTReader::Success:
+      // We successfully read a valid, up-to-date PCM.
+      // FIXME: This could update the timestamp. Regular calls to
+      // ASTReader::ReadAST() would do so unless they encountered corrupted
+      // AST block, corrupted extension block, or did not read the expected
+      // top-level module.
+      return;
+    case ASTReader::OutOfDate:
+    case ASTReader::Missing:
+      // The most interesting case.
+      break;
+    default:
+      // Let the regular scan diagnose this.
+      return;
+    }
+
+    ModCache.prepareForGetLock(ModuleFileName);
+    auto Lock = ModCache.getLock(ModuleFileName);
+    bool Owned;
+    llvm::Error LockErr = Lock->tryLock().moveInto(Owned);
+    // Someone else is building the PCM right now.
+    if (!LockErr && !Owned)
+      return;
+    // We should build the PCM.
+    // FIXME: Pass the correct BaseFS to the worker FS.
+    auto VFS = llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
+        Service.getSharedCache(), llvm::vfs::getRealFileSystem());
+    auto DC = std::make_unique<DiagnosticConsumer>();
+    auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries());
+    CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
+                                                        std::move(MC));
+    auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
+                                           CloneConfig);
+    auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
+                                           CloneConfig);
+
+    // FIXME: Have the service own a thread pool and use that instead.
+    // FIXME: This lock belongs to a module cache that might not outlive the
+    // thread. (This should work for now, because the in-process lock only
+    // refers to an object managed by the service, which should outlive this
+    // thread.)
+    std::thread([Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
+                 ModCI2 = std::move(ModCI2), DC = std::move(DC),
+                 Service = &Service] {
+      // Quickly discovers and compiles modules for the real scan below.
+      SingleModuleWithAsyncModuleCompiles Action1(*Service);
+      (void)ModCI1->ExecuteAction(Action1);
+      // The real scan below.
+      ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
+      GenerateModuleFromModuleMapAction Action2;
+      (void)ModCI2->ExecuteAction(Action2);
----------------
jansvoboda11 wrote:

Done.

https://github.com/llvm/llvm-project/pull/180047