r321864 - Add a tool executor that runs actions on all TUs in the compilation database.
Eric Liu via cfe-commits
cfe-commits at lists.llvm.org
Fri Jan 5 02:32:16 PST 2018
Author: ioeric
Date: Fri Jan 5 02:32:16 2018
New Revision: 321864
URL: http://llvm.org/viewvc/llvm-project?rev=321864&view=rev
Log:
Add a tool executor that runs actions on all TUs in the compilation database.
Summary: Tool results are deduplicated by the result key.
Reviewers: hokein
Subscribers: klimek, mgorny, cfe-commits
Differential Revision: https://reviews.llvm.org/D41729
Added:
cfe/trunk/include/clang/Tooling/AllTUsExecution.h
cfe/trunk/lib/Tooling/AllTUsExecution.cpp
Modified:
cfe/trunk/lib/Tooling/CMakeLists.txt
cfe/trunk/lib/Tooling/Execution.cpp
cfe/trunk/unittests/Tooling/ExecutionTest.cpp
Added: cfe/trunk/include/clang/Tooling/AllTUsExecution.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/AllTUsExecution.h?rev=321864&view=auto
==============================================================================
--- cfe/trunk/include/clang/Tooling/AllTUsExecution.h (added)
+++ cfe/trunk/include/clang/Tooling/AllTUsExecution.h Fri Jan 5 02:32:16 2018
@@ -0,0 +1,76 @@
+//===--- AllTUsExecution.h - Execute actions on all TUs. -*- C++ --------*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a tool executor that runs given actions on all TUs in the
+// compilation database. Tool results are deuplicated by the result key.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H
+#define LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H
+
+#include "clang/Tooling/ArgumentsAdjusters.h"
+#include "clang/Tooling/Execution.h"
+
+namespace clang {
+namespace tooling {
+
+/// \brief Executes given frontend actions on all files/TUs in the compilation
+/// database. The final results will be deduplicated by the result key.
+class AllTUsToolExecutor : public ToolExecutor {
+public:
+ static const char *ExecutorName;
+
+ /// \brief Init with \p CompilationDatabase.
+ /// This uses \p ThreadCount threads to exececute the actions on all files in
+ /// parallel. If \p ThreadCount is 0, this uses `llvm::hardware_concurrency`.
+ AllTUsToolExecutor(const CompilationDatabase &Compilations,
+ unsigned ThreadCount,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps =
+ std::make_shared<PCHContainerOperations>());
+
+ /// \brief Init with \p CommonOptionsParser. This is expected to be used by
+ /// `createExecutorFromCommandLineArgs` based on commandline options.
+ ///
+ /// The executor takes ownership of \p Options.
+ AllTUsToolExecutor(CommonOptionsParser Options, unsigned ThreadCount,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps =
+ std::make_shared<PCHContainerOperations>());
+
+ StringRef getExecutorName() const override { return ExecutorName; }
+
+ using ToolExecutor::execute;
+
+ llvm::Error
+ execute(llvm::ArrayRef<
+ std::pair<std::unique_ptr<FrontendActionFactory>, ArgumentsAdjuster>>
+ Actions) override;
+
+ ExecutionContext *getExecutionContext() override { return &Context; };
+
+ ToolResults *getToolResults() override { return Results.get(); }
+
+ void mapVirtualFile(StringRef FilePath, StringRef Content) override {
+ OverlayFiles[FilePath] = Content;
+ }
+
+private:
+ // Used to store the parser when the executor is initialized with parser.
+ llvm::Optional<CommonOptionsParser> OptionsParser;
+ const CompilationDatabase &Compilations;
+ std::unique_ptr<ToolResults> Results;
+ ExecutionContext Context;
+ llvm::StringMap<std::string> OverlayFiles;
+ unsigned ThreadCount;
+};
+
+} // end namespace tooling
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H
Added: cfe/trunk/lib/Tooling/AllTUsExecution.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/AllTUsExecution.cpp?rev=321864&view=auto
==============================================================================
--- cfe/trunk/lib/Tooling/AllTUsExecution.cpp (added)
+++ cfe/trunk/lib/Tooling/AllTUsExecution.cpp Fri Jan 5 02:32:16 2018
@@ -0,0 +1,165 @@
+//===- lib/Tooling/AllTUsExecution.cpp - Execute actions on all TUs. ------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/AllTUsExecution.h"
+#include "clang/Tooling/ToolExecutorPluginRegistry.h"
+#include "llvm/Support/ThreadPool.h"
+
+namespace clang {
+namespace tooling {
+
+const char *AllTUsToolExecutor::ExecutorName = "AllTUsToolExecutor";
+
+namespace {
+llvm::Error make_string_error(const llvm::Twine &Message) {
+ return llvm::make_error<llvm::StringError>(Message,
+ llvm::inconvertibleErrorCode());
+}
+
+ArgumentsAdjuster getDefaultArgumentsAdjusters() {
+ return combineAdjusters(
+ getClangStripOutputAdjuster(),
+ combineAdjusters(getClangSyntaxOnlyAdjuster(),
+ getClangStripDependencyFileAdjuster()));
+}
+
+class ThreadSafeToolResults : public ToolResults {
+public:
+ void addResult(StringRef Key, StringRef Value) override {
+ std::unique_lock<std::mutex> LockGuard(Mutex);
+ Results[Key] = Value;
+ }
+
+ std::vector<std::pair<std::string, std::string>> AllKVResults() override {
+ std::vector<std::pair<std::string, std::string>> KVs;
+ for (const auto &Pair : Results)
+ KVs.emplace_back(Pair.first().str(), Pair.second);
+ return KVs;
+ }
+
+ void forEachResult(llvm::function_ref<void(StringRef Key, StringRef Value)>
+ Callback) override {
+ for (const auto &Pair : Results)
+ Callback(Pair.first(), Pair.second);
+ }
+
+private:
+ llvm::StringMap<std::string> Results;
+ std::mutex Mutex;
+};
+
+} // namespace
+
+AllTUsToolExecutor::AllTUsToolExecutor(
+ const CompilationDatabase &Compilations, unsigned ThreadCount,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps)
+ : Compilations(Compilations), Results(new ThreadSafeToolResults),
+ Context(Results.get()), ThreadCount(ThreadCount) {}
+
+AllTUsToolExecutor::AllTUsToolExecutor(
+ CommonOptionsParser Options, unsigned ThreadCount,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps)
+ : OptionsParser(std::move(Options)),
+ Compilations(OptionsParser->getCompilations()),
+ Results(new ThreadSafeToolResults), Context(Results.get()),
+ ThreadCount(ThreadCount) {}
+
+llvm::Error AllTUsToolExecutor::execute(
+ llvm::ArrayRef<
+ std::pair<std::unique_ptr<FrontendActionFactory>, ArgumentsAdjuster>>
+ Actions) {
+ if (Actions.empty())
+ return make_string_error("No action to execute.");
+
+ if (Actions.size() != 1)
+ return make_string_error(
+ "Only support executing exactly 1 action at this point.");
+
+ std::string ErrorMsg;
+ std::mutex TUMutex;
+ auto AppendError = [&](llvm::Twine Err) {
+ std::unique_lock<std::mutex> LockGuard(TUMutex);
+ ErrorMsg += Err.str();
+ };
+
+ auto Log = [&](llvm::Twine Msg) {
+ std::unique_lock<std::mutex> LockGuard(TUMutex);
+ llvm::errs() << Msg.str() << "\n";
+ };
+
+ auto Files = Compilations.getAllFiles();
+ // Add a counter to track the progress.
+ const std::string TotalNumStr = std::to_string(Files.size());
+ unsigned Counter = 0;
+ auto Count = [&]() {
+ std::unique_lock<std::mutex> LockGuard(TUMutex);
+ return ++Counter;
+ };
+
+ auto &Action = Actions.front();
+
+ {
+ llvm::ThreadPool Pool(ThreadCount == 0 ? llvm::hardware_concurrency()
+ : ThreadCount);
+
+ for (std::string File : Files) {
+ Pool.async(
+ [&](std::string Path) {
+ Log("[" + std::to_string(Count()) + "/" + TotalNumStr +
+ "] Processing file " + Path);
+ ClangTool Tool(Compilations, {Path});
+ Tool.appendArgumentsAdjuster(Action.second);
+ Tool.appendArgumentsAdjuster(getDefaultArgumentsAdjusters());
+ for (const auto &FileAndContent : OverlayFiles)
+ Tool.mapVirtualFile(FileAndContent.first(),
+ FileAndContent.second);
+ if (Tool.run(Action.first.get()))
+ AppendError(llvm::Twine("Failed to run action on ") + Path +
+ "\n");
+ },
+ File);
+ }
+ }
+
+ if (!ErrorMsg.empty())
+ return make_string_error(ErrorMsg);
+
+ return llvm::Error::success();
+}
+
+static llvm::cl::opt<unsigned> ExecutorConcurrency(
+ "execute-concurrency",
+ llvm::cl::desc("The number of threads used to process all files in "
+ "parallel. Set to 0 for hardware concurrency."),
+ llvm::cl::init(0));
+
+class AllTUsToolExecutorPlugin : public ToolExecutorPlugin {
+public:
+ llvm::Expected<std::unique_ptr<ToolExecutor>>
+ create(CommonOptionsParser &OptionsParser) override {
+ if (OptionsParser.getSourcePathList().empty())
+ return make_string_error(
+ "[AllTUsToolExecutorPlugin] Please provide a directory/file path in "
+ "the compilation database.");
+ return llvm::make_unique<AllTUsToolExecutor>(std::move(OptionsParser),
+ ExecutorConcurrency);
+ }
+};
+
+static ToolExecutorPluginRegistry::Add<AllTUsToolExecutorPlugin>
+ X("all-TUs",
+ "Runs FrontendActions on all TUs in the compilation database. "
+ "Tool results are deduplicated by the result key and stored in memory.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the plugin.
+volatile int AllTUsToolExecutorAnchorSource = 0;
+
+} // end namespace tooling
+} // end namespace clang
Modified: cfe/trunk/lib/Tooling/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/CMakeLists.txt?rev=321864&r1=321863&r2=321864&view=diff
==============================================================================
--- cfe/trunk/lib/Tooling/CMakeLists.txt (original)
+++ cfe/trunk/lib/Tooling/CMakeLists.txt Fri Jan 5 02:32:16 2018
@@ -8,6 +8,7 @@ add_subdirectory(Refactoring)
add_subdirectory(ASTDiff)
add_clang_library(clangTooling
+ AllTUsExecution.cpp
ArgumentsAdjusters.cpp
CommonOptionsParser.cpp
CompilationDatabase.cpp
Modified: cfe/trunk/lib/Tooling/Execution.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Execution.cpp?rev=321864&r1=321863&r2=321864&view=diff
==============================================================================
--- cfe/trunk/lib/Tooling/Execution.cpp (original)
+++ cfe/trunk/lib/Tooling/Execution.cpp Fri Jan 5 02:32:16 2018
@@ -96,10 +96,13 @@ createExecutorFromCommandLineArgs(int &a
}
// This anchor is used to force the linker to link in the generated object file
-// and thus register the StandaloneToolExecutorPlugin.
+// and thus register the StandaloneToolExecutorPlugin etc.
extern volatile int StandaloneToolExecutorAnchorSource;
+extern volatile int AllTUsToolExecutorAnchorSource;
static int LLVM_ATTRIBUTE_UNUSED StandaloneToolExecutorAnchorDest =
StandaloneToolExecutorAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED AllTUsToolExecutorAnchorDest =
+ AllTUsToolExecutorAnchorSource;
} // end namespace tooling
} // end namespace clang
Modified: cfe/trunk/unittests/Tooling/ExecutionTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Tooling/ExecutionTest.cpp?rev=321864&r1=321863&r2=321864&view=diff
==============================================================================
--- cfe/trunk/unittests/Tooling/ExecutionTest.cpp (original)
+++ cfe/trunk/unittests/Tooling/ExecutionTest.cpp Fri Jan 5 02:32:16 2018
@@ -7,17 +7,19 @@
//
//===----------------------------------------------------------------------===//
+#include "clang/Tooling/Execution.h"
#include "clang/AST/ASTConsumer.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/Frontend/ASTUnit.h"
#include "clang/Frontend/FrontendAction.h"
#include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/AllTUsExecution.h"
#include "clang/Tooling/CompilationDatabase.h"
-#include "clang/Tooling/Execution.h"
#include "clang/Tooling/StandaloneExecution.h"
#include "clang/Tooling/ToolExecutorPluginRegistry.h"
#include "clang/Tooling/Tooling.h"
+#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include <algorithm>
#include <string>
@@ -217,5 +219,70 @@ TEST(StandaloneToolTest, SimpleActionWit
[](StringRef, StringRef Value) { EXPECT_EQ("1", Value); });
}
+class FixedCompilationDatabaseWithFiles : public CompilationDatabase {
+public:
+ FixedCompilationDatabaseWithFiles(Twine Directory,
+ ArrayRef<std::string> Files,
+ ArrayRef<std::string> CommandLine)
+ : FixedCompilations(Directory, CommandLine), Files(Files) {}
+
+ std::vector<CompileCommand>
+ getCompileCommands(StringRef FilePath) const override {
+ return FixedCompilations.getCompileCommands(FilePath);
+ }
+
+ std::vector<std::string> getAllFiles() const override { return Files; }
+
+private:
+ FixedCompilationDatabase FixedCompilations;
+ std::vector<std::string> Files;
+};
+
+MATCHER_P(Named, Name, "") { return arg.first == Name; }
+
+TEST(AllTUsToolTest, AFewFiles) {
+ FixedCompilationDatabaseWithFiles Compilations(".", {"a.cc", "b.cc", "c.cc"},
+ std::vector<std::string>());
+ AllTUsToolExecutor Executor(Compilations, /*ThreadCount=*/0);
+ Executor.mapVirtualFile("a.cc", "void x() {}");
+ Executor.mapVirtualFile("b.cc", "void y() {}");
+ Executor.mapVirtualFile("c.cc", "void z() {}");
+
+ auto Err = Executor.execute(std::unique_ptr<FrontendActionFactory>(
+ new ReportResultActionFactory(Executor.getExecutionContext())));
+ ASSERT_TRUE(!Err);
+ EXPECT_THAT(
+ Executor.getToolResults()->AllKVResults(),
+ ::testing::UnorderedElementsAre(Named("x"), Named("y"), Named("z")));
+}
+
+TEST(AllTUsToolTest, ManyFiles) {
+ unsigned NumFiles = 100;
+ std::vector<std::string> Files;
+ std::map<std::string, std::string> FileToContent;
+ std::vector<std::string> ExpectedSymbols;
+ for (unsigned i = 1; i <= NumFiles; ++i) {
+ std::string File = "f" + std::to_string(i) + ".cc";
+ std::string Symbol = "looong_function_name_" + std::to_string(i);
+ Files.push_back(File);
+ FileToContent[File] = "void " + Symbol + "() {}";
+ ExpectedSymbols.push_back(Symbol);
+ }
+ FixedCompilationDatabaseWithFiles Compilations(".", Files,
+ std::vector<std::string>());
+ AllTUsToolExecutor Executor(Compilations, /*ThreadCount=*/0);
+ for (const auto &FileAndContent : FileToContent) {
+ Executor.mapVirtualFile(FileAndContent.first, FileAndContent.second);
+ }
+
+ auto Err = Executor.execute(std::unique_ptr<FrontendActionFactory>(
+ new ReportResultActionFactory(Executor.getExecutionContext())));
+ ASSERT_TRUE(!Err);
+ std::vector<std::string> Results;
+ Executor.getToolResults()->forEachResult(
+ [&](StringRef Name, StringRef) { Results.push_back(Name); });
+ EXPECT_THAT(ExpectedSymbols, ::testing::UnorderedElementsAreArray(Results));
+}
+
} // end namespace tooling
} // end namespace clang
More information about the cfe-commits
mailing list