[clang] [clang][modules-driver] Add dependency scan and dependency graph (PR #152770)

[Chuanqi Xu via cfe-commits]

================
@@ -0,0 +1,1579 @@
+//===--- Driver.cpp - Clang GCC Compatible Driver -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines functionality to support driver managed builds for
+/// compilations which use Clang modules or standard C++20 named modules.
+///
+//===----------------------------------------------------------------------===//
+
+#include "clang/Driver/ModulesDriver.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticDriver.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Driver/Compilation.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/Job.h"
+#include "clang/Driver/Options.h"
+#include "clang/Driver/Tool.h"
+#include "clang/Lex/DependencyDirectivesScanner.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DirectedGraph.h"
+#include "llvm/ADT/GraphTraits.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Option/ArgList.h"
+#include "llvm/Option/Option.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/DOTGraphTraits.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <atomic>
+#include <iterator>
+#include <memory>
+#include <mutex>
+#include <numeric>
+#include <optional>
+#include <tuple>
+#include <utility>
+
+using namespace llvm::opt;
+
+namespace clang::driver::modules {
+using JobVector = JobList::list_type;
+
+// The tooling::deps namespace has conflicting names with clang::driver, we
+// therefore introduce only the required tooling::deps namespace members into
+// this namespace.
+using tooling::dependencies::DependencyActionController;
+using tooling::dependencies::DependencyScanningService;
+using tooling::dependencies::DependencyScanningWorker;
+using tooling::dependencies::FullDependencyConsumer;
+using tooling::dependencies::ModuleDeps;
+using tooling::dependencies::ModuleDepsGraph;
+using tooling::dependencies::ModuleID;
+using tooling::dependencies::ModuleOutputKind;
+using tooling::dependencies::ScanningMode;
+using tooling::dependencies::ScanningOutputFormat;
+using tooling::dependencies::TranslationUnitDeps;
+
+/// Returns true if any source input is of type c++-module.
+static bool hasCXXNamedModuleInput(const InputList &Inputs) {
+  const auto IsTypeCXXModule = [](const auto &Input) -> bool {
+    const auto TypeID = Input.first;
+    return (TypeID == types::TY_CXXModule);
+  };
+  return any_of(Inputs, IsTypeCXXModule);
+}
+
+/// Scan the leading lines of each C++ source file until C++20 named module
+/// usage is detected.
+///
+/// \returns true if module usage is detected, false otherwise, or a
+/// llvm::FileError on read failure.
+static Expected<bool> scanForCXXNamedModuleUsage(const InputList &Inputs,
+                                                 llvm::vfs::FileSystem &VFS,
+                                                 DiagnosticsEngine &Diags) {
+  const auto CXXInputs = make_filter_range(
+      Inputs, [](const InputTy &Input) { return types::isCXX(Input.first); });
+  for (const auto &Input : CXXInputs) {
+    auto Filename = Input.second->getSpelling();
+    auto MemBufOrErr = VFS.getBufferForFile(Filename);
+    if (!MemBufOrErr)
+      return llvm::createFileError(Filename, MemBufOrErr.getError());
+    const auto MemBuf = std::move(*MemBufOrErr);
+
+    // Scan the buffer using the dependency directives scanner.
+    if (clang::scanInputForCXXNamedModulesUsage(MemBuf->getBuffer())) {
+      Diags.Report(diag::remark_found_cxx20_module_usage) << Filename;
+      return true;
+    }
+  }
+  return false;
+}
+
+Expected<bool> shouldUseModulesDriver(const InputList &Inputs,
+                                      llvm::vfs::FileSystem &FS,
+                                      DiagnosticsEngine &Diags) {
+  if (Inputs.size() < 2)
+    return false;
+  if (hasCXXNamedModuleInput(Inputs))
+    return true;
+  return scanForCXXNamedModuleUsage(Inputs, FS, Diags);
+}
+
+static bool fromJSON(const llvm::json::Value &Params,
+                     StdModuleManifest::LocalModuleArgs &LocalArgs,
+                     llvm::json::Path P) {
+  llvm::json::ObjectMapper O(Params, P);
+  return O.mapOptional("system-include-directories",
+                       LocalArgs.SystemIncludeDirs);
+}
+
+static bool fromJSON(const llvm::json::Value &Params,
+                     StdModuleManifest::Module &ModuleEntry,
+                     llvm::json::Path P) {
+  llvm::json::ObjectMapper O(Params, P);
+  return O.map("is-std-library", ModuleEntry.IsStdlib) &&
+         O.map("logical-name", ModuleEntry.LogicalName) &&
+         O.map("source-path", ModuleEntry.SourcePath) &&
+         O.mapOptional("local-arguments", ModuleEntry.LocalArgs);
+}
+
+static bool fromJSON(const llvm::json::Value &Params,
+                     StdModuleManifest &Manifest, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(Params, P);
+  return O.map("modules", Manifest.ModuleEntries);
+}
+
+/// Parses the Standard library module manifest from \c Buffer.
+///
+/// The source file paths listed in the manifest are relative to its own
+/// path.
+static Expected<StdModuleManifest> parseStdModuleManifest(StringRef Buffer) {
+  auto ParsedJsonOrErr = llvm::json::parse(Buffer);
+  if (!ParsedJsonOrErr)
+    return ParsedJsonOrErr.takeError();
+
+  StdModuleManifest Manifest;
+  llvm::json::Path::Root Root;
+  if (!fromJSON(*ParsedJsonOrErr, Manifest, Root))
+    return Root.getError();
+
+  return Manifest;
+}
+
+/// Converts all file paths in \c Manifest from paths relative to
+/// \c ManifestPath (the manifest's location itself) to absolute.
+static void makeStdModuleManifestPathsAbsolute(StdModuleManifest &Manifest,
+                                               StringRef ManifestPath) {
+  SmallString<124> ManifestDir(ManifestPath);
+  llvm::sys::path::remove_filename(ManifestDir);
+
+  SmallString<256> TempPath;
+  auto ensureAbsolutePath = [&](std::string &Path) {
+    if (llvm::sys::path::is_absolute(Path))
+      return;
+    TempPath = ManifestDir;
+    llvm::sys::path::append(TempPath, Path);
+    llvm::sys::path::remove_dots(TempPath, true);
+    Path = std::string(TempPath);
+  };
+
+  for (auto &ModuleEntry : Manifest.ModuleEntries) {
+    ensureAbsolutePath(ModuleEntry.SourcePath);
+    if (!ModuleEntry.LocalArgs)
+      continue;
+    for (auto &IncludeDir : ModuleEntry.LocalArgs->SystemIncludeDirs)
+      ensureAbsolutePath(IncludeDir);
+  }
+}
+
+Expected<StdModuleManifest> readStdModuleManifest(StringRef ManifestPath,
+                                                  llvm::vfs::FileSystem &VFS) {
+  auto MemBufOrErr = VFS.getBufferForFile(ManifestPath);
+  if (!MemBufOrErr)
+    return llvm::createFileError(ManifestPath, MemBufOrErr.getError());
+  const auto MemBuf = std::move(*MemBufOrErr);
+
+  auto ManifestOrErr = parseStdModuleManifest(MemBuf->getBuffer());
+  if (!ManifestOrErr)
+    return ManifestOrErr.takeError();
+  auto Manifest = std::move(*ManifestOrErr);
+
+  // All paths in the manifest are relative to \c ManifestPath.
+  // Make them absolute.
+  makeStdModuleManifestPathsAbsolute(Manifest, ManifestPath);
+
+  return Manifest;
+}
+
+/// Appends a compilation input for the given \c Entry of the Standard library
+/// module manifest.
+static void
+appendStdModuleManifestInput(const StdModuleManifest::Module &ModuleEntry,
+                             Compilation &C, InputList &Inputs) {
+  auto &Args = C.getArgs();
+  const auto &Opts = C.getDriver().getOpts();
+
+  C.getDriver().DiagnoseInputExistence(Args, ModuleEntry.SourcePath,
+                                       types::TY_CXXModule,
+                                       /*TypoCorrect=*/false);
+
+  auto *A = new Arg(Opts.getOption(options::OPT_INPUT), ModuleEntry.SourcePath,
+                    Args.getBaseArgs().MakeIndex(ModuleEntry.SourcePath),
+                    Args.getBaseArgs().MakeArgString(ModuleEntry.SourcePath));
+  Args.AddSynthesizedArg(A);
+  A->claim();
+  Inputs.emplace_back(types::TY_CXXModule, A);
+}
+
+void buildStdModuleManifestInputs(const StdModuleManifest &Manifest,
----------------
ChuanqiXu9 wrote:

What is the status quo of distributed different std manifest files ?

https://github.com/llvm/llvm-project/pull/152770