r363204 - [clang-scan-deps] initial outline of the tool that runs preprocessor to find
Alex Lorenz via cfe-commits
cfe-commits at lists.llvm.org
Wed Jun 12 14:32:49 PDT 2019
Author: arphaman
Date: Wed Jun 12 14:32:49 2019
New Revision: 363204
URL: http://llvm.org/viewvc/llvm-project?rev=363204&view=rev
Log:
[clang-scan-deps] initial outline of the tool that runs preprocessor to find
dependencies over a JSON compilation database
This commit introduces an outline for the clang-scan-deps tool that will be
used to implement fast dependency discovery phase using implicit modules for
explicit module builds.
The initial version of the tool works by computing non-modular header dependencies
for files in the compilation database without any optimizations
(i.e. without source minimization from r362459).
The tool spawns a number of worker threads to run the clang compiler workers in parallel.
The immediate goal for clang-scan-deps is to create a ClangScanDeps library
which will be used to build up this tool to use the source minimization and
caching multi-threaded filesystem to implement the optimized non-incremental
dependency scanning phase for a non-modular build. This will allow us to do
benchmarks and comparisons for performance that the minimization and caching give us
Differential Revision: https://reviews.llvm.org/D60233
Added:
cfe/trunk/test/ClangScanDeps/
cfe/trunk/test/ClangScanDeps/Inputs/
cfe/trunk/test/ClangScanDeps/Inputs/header.h
cfe/trunk/test/ClangScanDeps/Inputs/header2.h
cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json
cfe/trunk/test/ClangScanDeps/regular_cdb.cpp
cfe/trunk/tools/clang-scan-deps/
cfe/trunk/tools/clang-scan-deps/CMakeLists.txt
cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp
Modified:
cfe/trunk/test/CMakeLists.txt
cfe/trunk/tools/CMakeLists.txt
Modified: cfe/trunk/test/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CMakeLists.txt?rev=363204&r1=363203&r2=363204&view=diff
==============================================================================
--- cfe/trunk/test/CMakeLists.txt (original)
+++ cfe/trunk/test/CMakeLists.txt Wed Jun 12 14:32:49 2019
@@ -57,6 +57,7 @@ list(APPEND CLANG_TEST_DEPS
clang-rename
clang-refactor
clang-diff
+ clang-scan-deps
diagtool
hmaptool
)
Added: cfe/trunk/test/ClangScanDeps/Inputs/header.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/Inputs/header.h?rev=363204&view=auto
==============================================================================
--- cfe/trunk/test/ClangScanDeps/Inputs/header.h (added)
+++ cfe/trunk/test/ClangScanDeps/Inputs/header.h Wed Jun 12 14:32:49 2019
@@ -0,0 +1,3 @@
+#ifdef INCLUDE_HEADER2
+#include "header2.h"
+#endif
Added: cfe/trunk/test/ClangScanDeps/Inputs/header2.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/Inputs/header2.h?rev=363204&view=auto
==============================================================================
--- cfe/trunk/test/ClangScanDeps/Inputs/header2.h (added)
+++ cfe/trunk/test/ClangScanDeps/Inputs/header2.h Wed Jun 12 14:32:49 2019
@@ -0,0 +1 @@
+// header 2.
Added: cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json?rev=363204&view=auto
==============================================================================
--- cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json (added)
+++ cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json Wed Jun 12 14:32:49 2019
@@ -0,0 +1,12 @@
+[
+{
+ "directory": "DIR",
+ "command": "clang -c DIR/regular_cdb.cpp -IInputs -MD -MF DIR/regular_cdb.d",
+ "file": "DIR/regular_cdb.cpp"
+},
+{
+ "directory": "DIR",
+ "command": "clang -c DIR/regular_cdb.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/regular_cdb2.d",
+ "file": "DIR/regular_cdb.cpp"
+}
+]
Added: cfe/trunk/test/ClangScanDeps/regular_cdb.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/regular_cdb.cpp?rev=363204&view=auto
==============================================================================
--- cfe/trunk/test/ClangScanDeps/regular_cdb.cpp (added)
+++ cfe/trunk/test/ClangScanDeps/regular_cdb.cpp Wed Jun 12 14:32:49 2019
@@ -0,0 +1,27 @@
+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.cdb
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/regular_cdb.cpp
+// RUN: mkdir %t.dir/Inputs
+// RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h
+// RUN: cp %S/Inputs/header2.h %t.dir/Inputs/header2.h
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/regular_cdb.json > %t.cdb
+//
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1
+// RUN: cat %t.dir/regular_cdb.d | FileCheck %s
+// RUN: cat %t.dir/regular_cdb2.d | FileCheck --check-prefix=CHECK2 %s
+// RUN: rm -rf %t.dir/regular_cdb.d %t.dir/regular_cdb2.d
+//
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 2
+// RUN: cat %t.dir/regular_cdb.d | FileCheck %s
+// RUN: cat %t.dir/regular_cdb2.d | FileCheck --check-prefix=CHECK2 %s
+
+#include "header.h"
+
+// CHECK: regular_cdb.cpp
+// CHECK-NEXT: Inputs{{/|\\}}header.h
+// CHECK-NOT: header2
+
+// CHECK2: regular_cdb.cpp
+// CHECK2-NEXT: Inputs{{/|\\}}header.h
+// CHECK2-NEXT: Inputs{{/|\\}}header2.h
Modified: cfe/trunk/tools/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CMakeLists.txt?rev=363204&r1=363203&r2=363204&view=diff
==============================================================================
--- cfe/trunk/tools/CMakeLists.txt (original)
+++ cfe/trunk/tools/CMakeLists.txt Wed Jun 12 14:32:49 2019
@@ -8,6 +8,7 @@ add_clang_subdirectory(clang-format-vs)
add_clang_subdirectory(clang-fuzzer)
add_clang_subdirectory(clang-import-test)
add_clang_subdirectory(clang-offload-bundler)
+add_clang_subdirectory(clang-scan-deps)
add_clang_subdirectory(c-index-test)
Added: cfe/trunk/tools/clang-scan-deps/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/clang-scan-deps/CMakeLists.txt?rev=363204&view=auto
==============================================================================
--- cfe/trunk/tools/clang-scan-deps/CMakeLists.txt (added)
+++ cfe/trunk/tools/clang-scan-deps/CMakeLists.txt Wed Jun 12 14:32:49 2019
@@ -0,0 +1,26 @@
+set(LLVM_LINK_COMPONENTS
+ Core
+ Support
+)
+
+add_clang_tool(clang-scan-deps
+ ClangScanDeps.cpp
+ )
+
+set(CLANG_SCAN_DEPS_LIB_DEPS
+ clangAST
+ clangBasic
+ clangCodeGen
+ clangDriver
+ clangFrontend
+ clangFrontendTool
+ clangLex
+ clangParse
+ clangTooling
+ )
+
+target_link_libraries(clang-scan-deps
+ PRIVATE
+ ${CLANG_SCAN_DEPS_LIB_DEPS}
+ )
+
Added: cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp?rev=363204&view=auto
==============================================================================
--- cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp (added)
+++ cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp Wed Jun 12 14:32:49 2019
@@ -0,0 +1,218 @@
+//===-- ClangScanDeps.cpp - Implementation of clang-scan-deps -------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/PCHContainerOperations.h"
+#include "clang/FrontendTool/Utils.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/JSONCompilationDatabase.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Options.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/Threading.h"
+#include <thread>
+
+using namespace clang;
+
+namespace {
+
+/// A clang tool that runs the preprocessor only for the given compiler
+/// invocation.
+class PreprocessorOnlyTool : public tooling::ToolAction {
+public:
+ PreprocessorOnlyTool(StringRef WorkingDirectory)
+ : WorkingDirectory(WorkingDirectory) {}
+
+ bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
+ FileManager *FileMgr,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+ DiagnosticConsumer *DiagConsumer) override {
+ // Create a compiler instance to handle the actual work.
+ CompilerInstance Compiler(std::move(PCHContainerOps));
+ Compiler.setInvocation(std::move(Invocation));
+ FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory;
+ Compiler.setFileManager(FileMgr);
+
+ // Create the compiler's actual diagnostics engine.
+ Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
+ if (!Compiler.hasDiagnostics())
+ return false;
+
+ Compiler.createSourceManager(*FileMgr);
+
+ auto Action = llvm::make_unique<PreprocessOnlyAction>();
+ const bool Result = Compiler.ExecuteAction(*Action);
+ FileMgr->clearStatCache();
+ return Result;
+ }
+
+private:
+ StringRef WorkingDirectory;
+};
+
+/// A proxy file system that doesn't call `chdir` when changing the working
+/// directory of a clang tool.
+class ProxyFileSystemWithoutChdir : public llvm::vfs::ProxyFileSystem {
+public:
+ ProxyFileSystemWithoutChdir(
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
+ : ProxyFileSystem(std::move(FS)) {}
+
+ llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
+ assert(!CWD.empty() && "empty CWD");
+ return CWD;
+ }
+
+ std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
+ CWD = Path.str();
+ return {};
+ }
+
+private:
+ std::string CWD;
+};
+
+/// The high-level implementation of the dependency discovery tool that runs on
+/// an individual worker thread.
+class DependencyScanningTool {
+public:
+ /// Construct a dependency scanning tool.
+ ///
+ /// \param Compilations The reference to the compilation database that's
+ /// used by the clang tool.
+ DependencyScanningTool(const tooling::CompilationDatabase &Compilations)
+ : Compilations(Compilations) {
+ PCHContainerOps = std::make_shared<PCHContainerOperations>();
+ BaseFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem());
+ }
+
+ /// Computes the dependencies for the given file.
+ ///
+ /// \returns True on error.
+ bool runOnFile(const std::string &Input, StringRef CWD) {
+ BaseFS->setCurrentWorkingDirectory(CWD);
+ tooling::ClangTool Tool(Compilations, Input, PCHContainerOps, BaseFS);
+ Tool.clearArgumentsAdjusters();
+ Tool.setRestoreWorkingDir(false);
+ PreprocessorOnlyTool Action(CWD);
+ return Tool.run(&Action);
+ }
+
+private:
+ const tooling::CompilationDatabase &Compilations;
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps;
+ /// The real filesystem used as a base for all the operations performed by the
+ /// tool.
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS;
+};
+
+llvm::cl::opt<bool> Help("h", llvm::cl::desc("Alias for -help"),
+ llvm::cl::Hidden);
+
+llvm::cl::OptionCategory DependencyScannerCategory("Tool options");
+
+llvm::cl::opt<unsigned>
+ NumThreads("j", llvm::cl::Optional,
+ llvm::cl::desc("Number of worker threads to use (default: use "
+ "all concurrent threads)"),
+ llvm::cl::init(0));
+
+llvm::cl::opt<std::string>
+ CompilationDB("compilation-database",
+ llvm::cl::desc("Compilation database"), llvm::cl::Required,
+ llvm::cl::cat(DependencyScannerCategory));
+
+} // end anonymous namespace
+
+int main(int argc, const char **argv) {
+ llvm::InitLLVM X(argc, argv);
+ llvm::cl::HideUnrelatedOptions(DependencyScannerCategory);
+ if (!llvm::cl::ParseCommandLineOptions(argc, argv))
+ return 1;
+
+ std::string ErrorMessage;
+ std::unique_ptr<tooling::JSONCompilationDatabase> Compilations =
+ tooling::JSONCompilationDatabase::loadFromFile(
+ CompilationDB, ErrorMessage,
+ tooling::JSONCommandLineSyntax::AutoDetect);
+ if (!Compilations) {
+ llvm::errs() << "error: " << ErrorMessage << "\n";
+ return 1;
+ }
+
+ llvm::cl::PrintOptionValues();
+
+ // By default the tool runs on all inputs in the CDB.
+ std::vector<std::pair<std::string, std::string>> Inputs;
+ for (const auto &Command : Compilations->getAllCompileCommands())
+ Inputs.emplace_back(Command.Filename, Command.Directory);
+
+ // The command options are rewritten to run Clang in preprocessor only mode.
+ auto AdjustingCompilations =
+ llvm::make_unique<tooling::ArgumentsAdjustingCompilations>(
+ std::move(Compilations));
+ AdjustingCompilations->appendArgumentsAdjuster(
+ [](const tooling::CommandLineArguments &Args, StringRef /*unused*/) {
+ tooling::CommandLineArguments AdjustedArgs = Args;
+ AdjustedArgs.push_back("-o");
+ AdjustedArgs.push_back("/dev/null");
+ AdjustedArgs.push_back("-Xclang");
+ AdjustedArgs.push_back("-Eonly");
+ AdjustedArgs.push_back("-Xclang");
+ AdjustedArgs.push_back("-sys-header-deps");
+ return AdjustedArgs;
+ });
+
+ unsigned NumWorkers =
+ NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;
+ std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
+ for (unsigned I = 0; I < NumWorkers; ++I)
+ WorkerTools.push_back(
+ llvm::make_unique<DependencyScanningTool>(*AdjustingCompilations));
+
+ std::vector<std::thread> WorkerThreads;
+ std::atomic<bool> HadErrors(false);
+ std::mutex Lock;
+ size_t Index = 0;
+
+ llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
+ << " files using " << NumWorkers << " workers\n";
+ for (unsigned I = 0; I < NumWorkers; ++I) {
+ WorkerThreads.emplace_back(
+ [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools]() {
+ while (true) {
+ std::string Input;
+ StringRef CWD;
+ // Take the next input.
+ {
+ std::unique_lock<std::mutex> LockGuard(Lock);
+ if (Index >= Inputs.size())
+ return;
+ const auto &Compilation = Inputs[Index++];
+ Input = Compilation.first;
+ CWD = Compilation.second;
+ }
+ // Run the tool on it.
+ if (WorkerTools[I]->runOnFile(Input, CWD))
+ HadErrors = true;
+ }
+ });
+ }
+ for (auto &W : WorkerThreads)
+ W.join();
+
+ return HadErrors;
+}
More information about the cfe-commits
mailing list