r194164 - Introduce ClangTool::buildASTs, and buildASTFromCode.
Sean Silva
chisophugis at gmail.com
Fri Jan 30 04:15:47 PST 2015
Wow, I'm just seeing this through its use on D7289. This is so great! Is
there some way we can draw more attention to this? Docs?
-- Sean Silva
On Wed, Nov 6, 2013 at 8:12 PM, Peter Collingbourne <peter at pcc.me.uk> wrote:
> Author: pcc
> Date: Wed Nov 6 14:12:45 2013
> New Revision: 194164
>
> URL: http://llvm.org/viewvc/llvm-project?rev=194164&view=rev
> Log:
> Introduce ClangTool::buildASTs, and buildASTFromCode.
>
> These allow clients to retrieve persistent AST objects (ASTUnits) which
> can be used in an ad-hoc manner after parsing.
>
> To accommodate this change, the code for processing a CompilerInvocation
> using a FrontendAction has been factored out to FrontendActionFactory, and
> a new base class, ToolAction, has been introduced, allowing the tool to do
> arbitrary things with each CompilerInvocation. This change was necessary
> because ASTUnit does not use the FrontendAction interface directly.
>
> This change also causes the FileManager in ClangTool to use shared
> ownership.
> This will become necessary because ASTUnit takes shared ownership of
> FileManager (ClangTool's FileManager is currently unused by ASTUnit; this
> is a FIXME). As shown in the tests, any client of ToolInvocation will
> need to be modified to use shared ownership for FileManager.
>
> Differential Revision: http://llvm-reviews.chandlerc.com/D2097
>
> Modified:
> cfe/trunk/include/clang/Tooling/Tooling.h
> cfe/trunk/lib/Tooling/Tooling.cpp
> cfe/trunk/unittests/Tooling/ToolingTest.cpp
>
> Modified: cfe/trunk/include/clang/Tooling/Tooling.h
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/Tooling.h?rev=194164&r1=194163&r2=194164&view=diff
>
> ==============================================================================
> --- cfe/trunk/include/clang/Tooling/Tooling.h (original)
> +++ cfe/trunk/include/clang/Tooling/Tooling.h Wed Nov 6 14:12:45 2013
> @@ -53,14 +53,33 @@ class FrontendAction;
>
> namespace tooling {
>
> +/// \brief Interface to process a clang::CompilerInvocation.
> +///
> +/// If your tool is based on FrontendAction, you should be deriving from
> +/// FrontendActionFactory instead.
> +class ToolAction {
> +public:
> + virtual ~ToolAction();
> +
> + /// \brief Perform an action for an invocation.
> + virtual bool runInvocation(clang::CompilerInvocation *Invocation,
> + FileManager *Files) = 0;
> +};
> +
> /// \brief Interface to generate clang::FrontendActions.
> ///
> /// Having a factory interface allows, for example, a new FrontendAction
> to be
> -/// created for each translation unit processed by ClangTool.
> -class FrontendActionFactory {
> +/// created for each translation unit processed by ClangTool. This class
> is
> +/// also a ToolAction which uses the FrontendActions created by create()
> to
> +/// process each translation unit.
> +class FrontendActionFactory : public ToolAction {
> public:
> virtual ~FrontendActionFactory();
>
> + /// \brief Invokes the compiler with a FrontendAction created by
> create().
> + bool runInvocation(clang::CompilerInvocation *Invocation,
> + FileManager *Files);
> +
> /// \brief Returns a new clang::FrontendAction.
> ///
> /// The caller takes ownership of the returned action.
> @@ -133,6 +152,26 @@ bool runToolOnCodeWithArgs(clang::Fronte
> const std::vector<std::string> &Args,
> const Twine &FileName = "input.cc");
>
> +/// \brief Builds an AST for 'Code'.
> +///
> +/// \param Code C++ code.
> +/// \param FileName The file name which 'Code' will be mapped as.
> +///
> +/// \return The resulting AST or null if an error occurred.
> +ASTUnit *buildASTFromCode(const Twine &Code,
> + const Twine &FileName = "input.cc");
> +
> +/// \brief Builds an AST for 'Code' with additional flags.
> +///
> +/// \param Code C++ code.
> +/// \param Args Additional flags to pass on.
> +/// \param FileName The file name which 'Code' will be mapped as.
> +///
> +/// \return The resulting AST or null if an error occurred.
> +ASTUnit *buildASTFromCodeWithArgs(const Twine &Code,
> + const std::vector<std::string> &Args,
> + const Twine &FileName = "input.cc");
> +
> /// \brief Utility to run a FrontendAction in a single clang invocation.
> class ToolInvocation {
> public:
> @@ -145,9 +184,19 @@ class ToolInvocation {
> /// \param ToolAction The action to be executed. Class takes ownership.
> /// \param Files The FileManager used for the execution. Class does not
> take
> /// ownership.
> - ToolInvocation(ArrayRef<std::string> CommandLine, FrontendAction
> *ToolAction,
> + ToolInvocation(ArrayRef<std::string> CommandLine, FrontendAction
> *FAction,
> FileManager *Files);
>
> + /// \brief Create a tool invocation.
> + ///
> + /// \param CommandLine The command line arguments to clang.
> + /// \param Action The action to be executed.
> + /// \param Files The FileManager used for the execution.
> + ToolInvocation(ArrayRef<std::string> CommandLine, ToolAction *Action,
> + FileManager *Files);
> +
> + ~ToolInvocation();
> +
> /// \brief Map a virtual file to be used while running the tool.
> ///
> /// \param FilePath The path at which the content will be mapped.
> @@ -167,7 +216,8 @@ class ToolInvocation {
> clang::CompilerInvocation *Invocation);
>
> std::vector<std::string> CommandLine;
> - OwningPtr<FrontendAction> ToolAction;
> + ToolAction *Action;
> + bool OwnsAction;
> FileManager *Files;
> // Maps <file name> -> <file content>.
> llvm::StringMap<StringRef> MappedFileContents;
> @@ -216,23 +266,25 @@ class ClangTool {
> /// \brief Clear the command line arguments adjuster chain.
> void clearArgumentsAdjusters();
>
> - /// Runs a frontend action over all files specified in the command line.
> + /// Runs an action over all files specified in the command line.
> ///
> - /// \param ActionFactory Factory generating the frontend actions. The
> function
> - /// takes ownership of this parameter. A new action is generated for
> every
> - /// processed translation unit.
> - virtual int run(FrontendActionFactory *ActionFactory);
> + /// \param Action Tool action.
> + int run(ToolAction *Action);
> +
> + /// \brief Create an AST for each file specified in the command line and
> + /// append them to ASTs.
> + int buildASTs(std::vector<ASTUnit *> &ASTs);
>
> /// \brief Returns the file manager used in the tool.
> ///
> /// The file manager is shared between all translation units.
> - FileManager &getFiles() { return Files; }
> + FileManager &getFiles() { return *Files; }
>
> private:
> // We store compile commands as pair (file name, compile command).
> std::vector< std::pair<std::string, CompileCommand> > CompileCommands;
>
> - FileManager Files;
> + llvm::IntrusiveRefCntPtr<FileManager> Files;
> // Contains a list of pairs (<file name>, <file content>).
> std::vector< std::pair<StringRef, StringRef> > MappedFileContents;
>
>
> Modified: cfe/trunk/lib/Tooling/Tooling.cpp
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Tooling.cpp?rev=194164&r1=194163&r2=194164&view=diff
>
> ==============================================================================
> --- cfe/trunk/lib/Tooling/Tooling.cpp (original)
> +++ cfe/trunk/lib/Tooling/Tooling.cpp Wed Nov 6 14:12:45 2013
> @@ -13,9 +13,11 @@
>
> //===----------------------------------------------------------------------===//
>
> #include "clang/Tooling/Tooling.h"
> +#include "clang/AST/ASTConsumer.h"
> #include "clang/Driver/Compilation.h"
> #include "clang/Driver/Driver.h"
> #include "clang/Driver/Tool.h"
> +#include "clang/Frontend/ASTUnit.h"
> #include "clang/Frontend/CompilerInstance.h"
> #include "clang/Frontend/FrontendDiagnostic.h"
> #include "clang/Frontend/TextDiagnosticPrinter.h"
> @@ -38,6 +40,8 @@
> namespace clang {
> namespace tooling {
>
> +ToolAction::~ToolAction() {}
> +
> FrontendActionFactory::~FrontendActionFactory() {}
>
> // FIXME: This file contains structural duplication with other parts of
> the
> @@ -104,18 +108,26 @@ bool runToolOnCode(clang::FrontendAction
> ToolAction, Code, std::vector<std::string>(), FileName);
> }
>
> +static std::vector<std::string>
> +getSyntaxOnlyToolArgs(const std::vector<std::string> &ExtraArgs,
> + StringRef FileName) {
> + std::vector<std::string> Args;
> + Args.push_back("clang-tool");
> + Args.push_back("-fsyntax-only");
> + Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end());
> + Args.push_back(FileName.str());
> + return Args;
> +}
> +
> bool runToolOnCodeWithArgs(clang::FrontendAction *ToolAction, const Twine
> &Code,
> const std::vector<std::string> &Args,
> const Twine &FileName) {
> SmallString<16> FileNameStorage;
> StringRef FileNameRef =
> FileName.toNullTerminatedStringRef(FileNameStorage);
> - std::vector<std::string> Commands;
> - Commands.push_back("clang-tool");
> - Commands.push_back("-fsyntax-only");
> - Commands.insert(Commands.end(), Args.begin(), Args.end());
> - Commands.push_back(FileNameRef.data());
> - FileManager Files((FileSystemOptions()));
> - ToolInvocation Invocation(Commands, ToolAction, &Files);
> + llvm::IntrusiveRefCntPtr<FileManager> Files(
> + new FileManager(FileSystemOptions()));
> + ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef),
> ToolAction,
> + Files.getPtr());
>
> SmallString<1024> CodeStorage;
> Invocation.mapVirtualFile(FileNameRef,
> @@ -138,10 +150,33 @@ std::string getAbsolutePath(StringRef Fi
> return AbsolutePath.str();
> }
>
> -ToolInvocation::ToolInvocation(
> - ArrayRef<std::string> CommandLine, FrontendAction *ToolAction,
> - FileManager *Files)
> - : CommandLine(CommandLine.vec()), ToolAction(ToolAction),
> Files(Files) {
> +namespace {
> +
> +class SingleFrontendActionFactory : public FrontendActionFactory {
> + FrontendAction *Action;
> +
> +public:
> + SingleFrontendActionFactory(FrontendAction *Action) : Action(Action) {}
> +
> + FrontendAction *create() { return Action; }
> +};
> +
> +}
> +
> +ToolInvocation::ToolInvocation(ArrayRef<std::string> CommandLine,
> + ToolAction *Action, FileManager *Files)
> + : CommandLine(CommandLine.vec()), Action(Action), OwnsAction(false),
> + Files(Files) {}
> +
> +ToolInvocation::ToolInvocation(ArrayRef<std::string> CommandLine,
> + FrontendAction *FAction, FileManager
> *Files)
> + : CommandLine(CommandLine.vec()),
> + Action(new SingleFrontendActionFactory(FAction)), OwnsAction(true),
> + Files(Files) {}
> +
> +ToolInvocation::~ToolInvocation() {
> + if (OwnsAction)
> + delete Action;
> }
>
> void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef
> Content) {
> @@ -175,6 +210,14 @@ bool ToolInvocation::run() {
> }
> OwningPtr<clang::CompilerInvocation> Invocation(
> newInvocation(&Diagnostics, *CC1Args));
> + for (llvm::StringMap<StringRef>::const_iterator
> + It = MappedFileContents.begin(), End =
> MappedFileContents.end();
> + It != End; ++It) {
> + // Inject the code as the given file name into the preprocessor
> options.
> + const llvm::MemoryBuffer *Input =
> + llvm::MemoryBuffer::getMemBuffer(It->getValue());
> + Invocation->getPreprocessorOpts().addRemappedFile(It->getKey(),
> Input);
> + }
> return runInvocation(BinaryName, Compilation.get(), Invocation.take());
> }
>
> @@ -189,16 +232,20 @@ bool ToolInvocation::runInvocation(
> llvm::errs() << "\n";
> }
>
> + return Action->runInvocation(Invocation, Files);
> +}
> +
> +bool FrontendActionFactory::runInvocation(CompilerInvocation *Invocation,
> + FileManager *Files) {
> // Create a compiler instance to handle the actual work.
> clang::CompilerInstance Compiler;
> Compiler.setInvocation(Invocation);
> Compiler.setFileManager(Files);
> - // FIXME: What about LangOpts?
>
> - // ToolAction can have lifetime requirements for Compiler or its
> members, and
> - // we need to ensure it's deleted earlier than Compiler. So we pass it
> to an
> - // OwningPtr declared after the Compiler variable.
> - OwningPtr<FrontendAction> ScopedToolAction(ToolAction.take());
> + // The FrontendAction can have lifetime requirements for Compiler or its
> + // members, and we need to ensure it's deleted earlier than Compiler.
> So we
> + // pass it to an OwningPtr declared after the Compiler variable.
> + OwningPtr<FrontendAction> ScopedToolAction(create());
>
> // Create the compilers actual diagnostics engine.
> Compiler.createDiagnostics();
> @@ -206,32 +253,16 @@ bool ToolInvocation::runInvocation(
> return false;
>
> Compiler.createSourceManager(*Files);
> - addFileMappingsTo(Compiler.getSourceManager());
>
> const bool Success = Compiler.ExecuteAction(*ScopedToolAction);
>
> - Compiler.resetAndLeakFileManager();
> Files->clearStatCaches();
> return Success;
> }
>
> -void ToolInvocation::addFileMappingsTo(SourceManager &Sources) {
> - for (llvm::StringMap<StringRef>::const_iterator
> - It = MappedFileContents.begin(), End =
> MappedFileContents.end();
> - It != End; ++It) {
> - // Inject the code as the given file name into the preprocessor
> options.
> - const llvm::MemoryBuffer *Input =
> - llvm::MemoryBuffer::getMemBuffer(It->getValue());
> - // FIXME: figure out what '0' stands for.
> - const FileEntry *FromFile = Files->getVirtualFile(
> - It->getKey(), Input->getBufferSize(), 0);
> - Sources.overrideFileContents(FromFile, Input);
> - }
> -}
> -
> ClangTool::ClangTool(const CompilationDatabase &Compilations,
> ArrayRef<std::string> SourcePaths)
> - : Files((FileSystemOptions())) {
> + : Files(new FileManager(FileSystemOptions())) {
> ArgsAdjusters.push_back(new ClangStripOutputAdjuster());
> ArgsAdjusters.push_back(new ClangSyntaxOnlyAdjuster());
> for (unsigned I = 0, E = SourcePaths.size(); I != E; ++I) {
> @@ -274,7 +305,7 @@ void ClangTool::clearArgumentsAdjusters(
> ArgsAdjusters.clear();
> }
>
> -int ClangTool::run(FrontendActionFactory *ActionFactory) {
> +int ClangTool::run(ToolAction *Action) {
> // Exists solely for the purpose of lookup of the resource path.
> // This just needs to be some symbol in the binary.
> static int StaticSymbol;
> @@ -309,7 +340,7 @@ int ClangTool::run(FrontendActionFactory
> DEBUG({
> llvm::dbgs() << "Processing: " << File << ".\n";
> });
> - ToolInvocation Invocation(CommandLine, ActionFactory->create(),
> &Files);
> + ToolInvocation Invocation(CommandLine, Action, Files.getPtr());
> for (int I = 0, E = MappedFileContents.size(); I != E; ++I) {
> Invocation.mapVirtualFile(MappedFileContents[I].first,
> MappedFileContents[I].second);
> @@ -323,5 +354,58 @@ int ClangTool::run(FrontendActionFactory
> return ProcessingFailed ? 1 : 0;
> }
>
> +namespace {
> +
> +class ASTBuilderAction : public ToolAction {
> + std::vector<ASTUnit *> &ASTs;
> +
> +public:
> + ASTBuilderAction(std::vector<ASTUnit *> &ASTs) : ASTs(ASTs) {}
> +
> + bool runInvocation(CompilerInvocation *Invocation,
> + FileManager *Files) {
> + // FIXME: This should use the provided FileManager.
> + ASTUnit *AST = ASTUnit::LoadFromCompilerInvocation(
> + Invocation,
> +
> CompilerInstance::createDiagnostics(&Invocation->getDiagnosticOpts()));
> + if (!AST)
> + return false;
> +
> + ASTs.push_back(AST);
> + return true;
> + }
> +};
> +
> +}
> +
> +int ClangTool::buildASTs(std::vector<ASTUnit *> &ASTs) {
> + ASTBuilderAction Action(ASTs);
> + return run(&Action);
> +}
> +
> +ASTUnit *buildASTFromCode(const Twine &Code, const Twine &FileName) {
> + return buildASTFromCodeWithArgs(Code, std::vector<std::string>(),
> FileName);
> +}
> +
> +ASTUnit *buildASTFromCodeWithArgs(const Twine &Code,
> + const std::vector<std::string> &Args,
> + const Twine &FileName) {
> + SmallString<16> FileNameStorage;
> + StringRef FileNameRef =
> FileName.toNullTerminatedStringRef(FileNameStorage);
> +
> + std::vector<ASTUnit *> ASTs;
> + ASTBuilderAction Action(ASTs);
> + ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef),
> &Action, 0);
> +
> + SmallString<1024> CodeStorage;
> + Invocation.mapVirtualFile(FileNameRef,
> + Code.toNullTerminatedStringRef(CodeStorage));
> + if (!Invocation.run())
> + return 0;
> +
> + assert(ASTs.size() == 1);
> + return ASTs[0];
> +}
> +
> } // end namespace tooling
> } // end namespace clang
>
> Modified: cfe/trunk/unittests/Tooling/ToolingTest.cpp
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Tooling/ToolingTest.cpp?rev=194164&r1=194163&r2=194164&view=diff
>
> ==============================================================================
> --- cfe/trunk/unittests/Tooling/ToolingTest.cpp (original)
> +++ cfe/trunk/unittests/Tooling/ToolingTest.cpp Wed Nov 6 14:12:45 2013
> @@ -10,12 +10,14 @@
> #include "clang/AST/ASTConsumer.h"
> #include "clang/AST/DeclCXX.h"
> #include "clang/AST/DeclGroup.h"
> +#include "clang/Frontend/ASTUnit.h"
> #include "clang/Frontend/CompilerInstance.h"
> #include "clang/Frontend/FrontendAction.h"
> #include "clang/Frontend/FrontendActions.h"
> #include "clang/Tooling/CompilationDatabase.h"
> #include "clang/Tooling/Tooling.h"
> #include "gtest/gtest.h"
> +#include "llvm/ADT/STLExtras.h"
> #include <string>
>
> namespace clang {
> @@ -83,6 +85,18 @@ class FindClassDeclXConsumer : public cl
> private:
> bool *FoundClassDeclX;
> };
> +bool FindClassDeclX(ASTUnit *AST) {
> + for (std::vector<Decl *>::iterator i = AST->top_level_begin(),
> + e = AST->top_level_end();
> + i != e; ++i) {
> + if (CXXRecordDecl* Record = dyn_cast<clang::CXXRecordDecl>(*i)) {
> + if (Record->getName() == "X") {
> + return true;
> + }
> + }
> + }
> + return false;
> +}
> } // end namespace
>
> TEST(runToolOnCode, FindsClassDecl) {
> @@ -97,6 +111,16 @@ TEST(runToolOnCode, FindsClassDecl) {
> EXPECT_FALSE(FoundClassDeclX);
> }
>
> +TEST(buildASTFromCode, FindsClassDecl) {
> + OwningPtr<ASTUnit> AST(buildASTFromCode("class X;"));
> + ASSERT_TRUE(AST.get());
> + EXPECT_TRUE(FindClassDeclX(AST.get()));
> +
> + AST.reset(buildASTFromCode("class Y;"));
> + ASSERT_TRUE(AST.get());
> + EXPECT_FALSE(FindClassDeclX(AST.get()));
> +}
> +
> TEST(newFrontendActionFactory, CreatesFrontendActionFactoryFromType) {
> OwningPtr<FrontendActionFactory> Factory(
> newFrontendActionFactory<SyntaxOnlyAction>());
> @@ -119,13 +143,15 @@ TEST(newFrontendActionFactory, CreatesFr
> }
>
> TEST(ToolInvocation, TestMapVirtualFile) {
> - clang::FileManager Files((clang::FileSystemOptions()));
> + IntrusiveRefCntPtr<clang::FileManager> Files(
> + new clang::FileManager(clang::FileSystemOptions()));
> std::vector<std::string> Args;
> Args.push_back("tool-executable");
> Args.push_back("-Idef");
> Args.push_back("-fsyntax-only");
> Args.push_back("test.cpp");
> - clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction,
> &Files);
> + clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction,
> + Files.getPtr());
> Invocation.mapVirtualFile("test.cpp", "#include <abc>\n");
> Invocation.mapVirtualFile("def/abc", "\n");
> EXPECT_TRUE(Invocation.run());
> @@ -136,13 +162,15 @@ TEST(ToolInvocation, TestVirtualModulesC
> // mapped module.map is found on the include path. In the future,
> expand this
> // test to run a full modules enabled compilation, so we make sure we
> can
> // rerun modules compilations with a virtual file system.
> - clang::FileManager Files((clang::FileSystemOptions()));
> + IntrusiveRefCntPtr<clang::FileManager> Files(
> + new clang::FileManager(clang::FileSystemOptions()));
> std::vector<std::string> Args;
> Args.push_back("tool-executable");
> Args.push_back("-Idef");
> Args.push_back("-fsyntax-only");
> Args.push_back("test.cpp");
> - clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction,
> &Files);
> + clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction,
> + Files.getPtr());
> Invocation.mapVirtualFile("test.cpp", "#include <abc>\n");
> Invocation.mapVirtualFile("def/abc", "\n");
> // Add a module.map file in the include directory of our header, so we
> trigger
> @@ -254,5 +282,23 @@ TEST(ClangToolTest, ArgumentAdjusters) {
> EXPECT_FALSE(Found);
> }
>
> +TEST(ClangToolTest, BuildASTs) {
> + FixedCompilationDatabase Compilations("/", std::vector<std::string>());
> +
> + std::vector<std::string> Sources;
> + Sources.push_back("/a.cc");
> + Sources.push_back("/b.cc");
> + ClangTool Tool(Compilations, Sources);
> +
> + Tool.mapVirtualFile("/a.cc", "void a() {}");
> + Tool.mapVirtualFile("/b.cc", "void b() {}");
> +
> + std::vector<ASTUnit *> ASTs;
> + EXPECT_EQ(0, Tool.buildASTs(ASTs));
> + EXPECT_EQ(2u, ASTs.size());
> +
> + llvm::DeleteContainerPointers(ASTs);
> +}
> +
> } // end namespace tooling
> } // end namespace clang
>
>
> _______________________________________________
> cfe-commits mailing list
> cfe-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20150130/0f6ae146/attachment.html>
More information about the cfe-commits
mailing list