[clang] [clang][DependencyScanning] Implementation of `CompilerInstanceWithContext` to Improve By-Name Queries (PR #164345)

[Jan Svoboda via cfe-commits]

================
@@ -705,3 +707,185 @@ bool DependencyScanningAction::runInvocation(
 
   return Result;
 }
+
+const std::string CompilerInstanceWithContext::FakeFileBuffer =
+    std::string(MAX_NUM_NAMES, ' ');
+
+llvm::Error CompilerInstanceWithContext::initialize() {
+  std::tie(OverlayFS, CommandLine) = initVFSForByNameScanning(
+      Worker.BaseFS, CommandLine, CWD, "ScanningByName");
+
+  DiagPrinterWithOS =
+      std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine);
+  DiagEngineWithCmdAndOpts = std::make_unique<DignosticsEngineWithDiagOpts>(
+      CommandLine, OverlayFS, DiagPrinterWithOS->DiagPrinter);
+
+  std::tie(Driver, Compilation) = buildCompilation(
+      CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, OverlayFS);
+
+  if (!Compilation) {
+    return llvm::make_error<llvm::StringError>("Failed to build compilation",
+                                               llvm::inconvertibleErrorCode());
+  }
+
+  const driver::Command &Command = *(Compilation->getJobs().begin());
+  const auto &CommandArgs = Command.getArguments();
+  size_t ArgSize = CommandArgs.size();
+  assert(ArgSize >= 1 && "Cannot have a command with 0 args");
+  const char *FirstArg = CommandArgs[0];
+  if (strcmp(FirstArg, "-cc1"))
+    return llvm::make_error<llvm::StringError>(
+        "Incorrect compilation command, missing cc1",
+        llvm::inconvertibleErrorCode());
+  Invocation = std::make_unique<CompilerInvocation>();
+
+  if (!CompilerInvocation::CreateFromArgs(*Invocation, Command.getArguments(),
+                                          *DiagEngineWithCmdAndOpts->DiagEngine,
+                                          Command.getExecutable())) {
+    DiagEngineWithCmdAndOpts->DiagEngine->Report(
+        diag::err_fe_expected_compiler_job)
+        << llvm::join(CommandLine, " ");
+    return llvm::make_error<llvm::StringError>(
+        "Cannot create CompilerInvocation from Args",
+        llvm::inconvertibleErrorCode());
+  }
+
+  // TODO: CMDArgsStrVector is making string copies. We should optimize later
+  // and avoid the copies.
+  std::vector<std::string> CMDArgsStrVector(ArgSize + 1);
+  CMDArgsStrVector.push_back(Command.getExecutable());
+  llvm::transform(CommandArgs, CMDArgsStrVector.begin() + 1,
+                  [](const char *s) { return std::string(s); });
+
+  Invocation = createCompilerInvocation(CMDArgsStrVector,
+                                        *DiagEngineWithCmdAndOpts->DiagEngine);
+  if (!Invocation) {
+    DiagEngineWithCmdAndOpts->DiagEngine->Report(
+        diag::err_fe_expected_compiler_job)
+        << llvm::join(CommandLine, " ");
+    return llvm::make_error<llvm::StringError>(
+        "Cannot create CompilerInvocation from Args",
+        llvm::inconvertibleErrorCode());
+  }
+
+  Invocation->getFrontendOpts().DisableFree = false;
+  Invocation->getCodeGenOpts().DisableFree = false;
+
+  if (any(Worker.Service.getOptimizeArgs() & ScanningOptimizations::Macros))
+    canonicalizeDefines(Invocation->getPreprocessorOpts());
+
+  // Create the CompilerInstance.
+  IntrusiveRefCntPtr<ModuleCache> ModCache =
+      makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());
+  CIPtr = std::make_unique<CompilerInstance>(
+      std::make_shared<CompilerInvocation>(*Invocation), Worker.PCHContainerOps,
+      ModCache.get());
+  auto &CI = *CIPtr;
+
+  if (!initializeScanCompilerInstance(
+          CI, OverlayFS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(),
+          Worker.Service, Worker.DepFS)) {
+    return llvm::make_error<llvm::StringError>(
+        "Cannot initialize scanning compiler instance",
+        llvm::inconvertibleErrorCode());
+  }
+
+  llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(CI);
+  auto MaybePrebuiltModulesASTMap =
+      computePrebuiltModulesASTMap(CI, StableDirs);
+  if (!MaybePrebuiltModulesASTMap)
+    return llvm::make_error<llvm::StringError>(
+        "Prebuilt module scanning failed", llvm::inconvertibleErrorCode());
+
+  PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
+  OutputOpts = takeDependencyOutputOptionsFrom(CI);
+
+  CI.createTarget();
+  // CI.initializeDelayedInputFileFromCAS();
+
+  return llvm::Error::success();
+}
+
+llvm::Error CompilerInstanceWithContext::computeDependencies(
+    StringRef ModuleName, DependencyConsumer &Consumer,
+    DependencyActionController &Controller) {
+  auto &CI = *CIPtr;
+  CompilerInvocation Inv(*Invocation);
+
+  CI.clearDependencyCollectors();
+  auto MDC = initializeScanInstanceDependencyCollector(
+      CI, std::make_unique<DependencyOutputOptions>(*OutputOpts), CWD, Consumer,
+      Worker.Service, *Invocation, Controller, PrebuiltModuleASTMap,
+      StableDirs);
+
+  if (!SrcLocOffset) {
+    // When SrcLocOffset is zero, we are at the beginning of the fake source
+    // file. In this case, we call BeginSourceFile to initialize.
+    std::unique_ptr<FrontendAction> Action =
+        std::make_unique<GetDependenciesByModuleNameAction>(ModuleName);
+    auto InputFile = CI.getFrontendOpts().Inputs.begin();
+
+    Action->BeginSourceFile(CI, *InputFile);
+  }
+
+  Preprocessor &PP = CI.getPreprocessor();
+  SourceManager &SM = PP.getSourceManager();
+  FileID MainFileID = SM.getMainFileID();
+  SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
+  SourceLocation IDLocation = FileStart.getLocWithOffset(SrcLocOffset);
+  if (!SrcLocOffset) {
+    // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
+    // the preprocessor.
+    PP.EnterSourceFile(MainFileID, nullptr, SourceLocation());
+  } else {
+    // When SrcLocOffset is non-zero, the preprocessor has already been
+    // initialized through a previous call of computeDependencies. We want to
+    // preserve the PP's state, hence we do not call EnterSourceFile again.
+    auto DCs = CI.getDependencyCollectors();
+    for (auto &DC : DCs) {
+      DC->attachToPreprocessor(PP);
+      auto *CB = DC->getPPCallbacks();
+
+      FileID PrevFID;
+      SrcMgr::CharacteristicKind FileType =
+          SM.getFileCharacteristic(IDLocation);
+      CB->LexedFileChanged(MainFileID,
+                           PPChainedCallbacks::LexedFileChangeReason::EnterFile,
+                           FileType, PrevFID, IDLocation);
+    }
+  }
+
+  SrcLocOffset++;
+  SmallVector<IdentifierLoc, 2> Path;
+  IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
+  Path.emplace_back(IDLocation, ModuleID);
+  auto ModResult = CI.loadModule(IDLocation, Path, Module::Hidden, false);
+
+  auto DCs = CI.getDependencyCollectors();
+  for (auto &DC : DCs) {
+    auto *CB = DC->getPPCallbacks();
+    assert(CB && "DC must have dependency collector callback");
+    CB->moduleImport(SourceLocation(), Path, ModResult);
+    CB->EndOfMainFile();
+  }
+
+  MDC->applyDiscoveredDependencies(Inv);
+  Consumer.handleBuildCommand({CommandLine[0], Inv.getCC1CommandLine()});
----------------
jansvoboda11 wrote:

Won't this have the change to `DisableFree` flags we did in `CompilerInstanceWithContext::initialize()`?

https://github.com/llvm/llvm-project/pull/164345