[clang-tools-extra] r320486 - [clangd] Introduce a "Symbol" class.
Evgenii Stepanov via cfe-commits
cfe-commits at lists.llvm.org
Tue Dec 12 17:09:12 PST 2017
Hi,
this does not compile for me because of a tag mismatch:
clang-tools-extra/clangd/index/Index.h:52:10: error: class
'DenseMapInfo' was previously declared as a struct
[-Werror,-Wmismatched-tags]
friend class llvm::DenseMapInfo<clang::clangd::SymbolID>;
On Tue, Dec 12, 2017 at 7:42 AM, Haojian Wu via cfe-commits
<cfe-commits at lists.llvm.org> wrote:
> Author: hokein
> Date: Tue Dec 12 07:42:10 2017
> New Revision: 320486
>
> URL: http://llvm.org/viewvc/llvm-project?rev=320486&view=rev
> Log:
> [clangd] Introduce a "Symbol" class.
>
> Summary:
> * The "Symbol" class represents a C++ symbol in the codebase, containing all the
> information of a C++ symbol needed by clangd. clangd will use it in clangd's
> AST/dynamic index and global/static index (code completion and code
> navigation).
> * The SymbolCollector (another IndexAction) will be used to recollect the
> symbols when the source file is changed (for ASTIndex), or to generate
> all C++ symbols for the whole project.
>
> In the long term (when index-while-building is ready), clangd should share a
> same "Symbol" structure and IndexAction with index-while-building, but
> for now we want to have some stuff working in clangd.
>
> Reviewers: ioeric, sammccall, ilya-biryukov, malaperle
>
> Reviewed By: sammccall
>
> Subscribers: malaperle, klimek, mgorny, cfe-commits
>
> Differential Revision: https://reviews.llvm.org/D40897
>
> Added:
> clang-tools-extra/trunk/clangd/index/
> clang-tools-extra/trunk/clangd/index/Index.cpp
> clang-tools-extra/trunk/clangd/index/Index.h
> clang-tools-extra/trunk/clangd/index/SymbolCollector.cpp
> clang-tools-extra/trunk/clangd/index/SymbolCollector.h
> clang-tools-extra/trunk/unittests/clangd/SymbolCollectorTests.cpp
> Modified:
> clang-tools-extra/trunk/clangd/CMakeLists.txt
> clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt
>
> Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=320486&r1=320485&r2=320486&view=diff
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
> +++ clang-tools-extra/trunk/clangd/CMakeLists.txt Tue Dec 12 07:42:10 2017
> @@ -19,6 +19,8 @@ add_clang_library(clangDaemon
> Protocol.cpp
> ProtocolHandlers.cpp
> Trace.cpp
> + index/Index.cpp
> + index/SymbolCollector.cpp
>
> LINK_LIBS
> clangAST
>
> Added: clang-tools-extra/trunk/clangd/index/Index.cpp
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=320486&view=auto
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/Index.cpp (added)
> +++ clang-tools-extra/trunk/clangd/index/Index.cpp Tue Dec 12 07:42:10 2017
> @@ -0,0 +1,49 @@
> +//===--- Index.cpp -----------------------------------------------*- C++-*-===//
> +//
> +// The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +
> +#include "Index.h"
> +
> +#include "llvm/Support/SHA1.h"
> +
> +namespace clang {
> +namespace clangd {
> +
> +namespace {
> +ArrayRef<uint8_t> toArrayRef(StringRef S) {
> + return {reinterpret_cast<const uint8_t *>(S.data()), S.size()};
> +}
> +} // namespace
> +
> +SymbolID::SymbolID(llvm::StringRef USR)
> + : HashValue(llvm::SHA1::hash(toArrayRef(USR))) {}
> +
> +SymbolSlab::const_iterator SymbolSlab::begin() const {
> + return Symbols.begin();
> +}
> +
> +SymbolSlab::const_iterator SymbolSlab::end() const {
> + return Symbols.end();
> +}
> +
> +SymbolSlab::const_iterator SymbolSlab::find(const SymbolID& SymID) const {
> + return Symbols.find(SymID);
> +}
> +
> +void SymbolSlab::freeze() {
> + Frozen = true;
> +}
> +
> +void SymbolSlab::insert(Symbol S) {
> + assert(!Frozen &&
> + "Can't insert a symbol after the slab has been frozen!");
> + Symbols[S.ID] = std::move(S);
> +}
> +
> +} // namespace clangd
> +} // namespace clang
>
> Added: clang-tools-extra/trunk/clangd/index/Index.h
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=320486&view=auto
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/Index.h (added)
> +++ clang-tools-extra/trunk/clangd/index/Index.h Tue Dec 12 07:42:10 2017
> @@ -0,0 +1,136 @@
> +//===--- Symbol.h -----------------------------------------------*- C++-*-===//
> +//
> +// The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===---------------------------------------------------------------------===//
> +
> +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
> +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
> +
> +#include "clang/Index/IndexSymbol.h"
> +#include "llvm/ADT/DenseMap.h"
> +#include "llvm/ADT/StringExtras.h"
> +
> +#include <array>
> +#include <string>
> +
> +namespace clang {
> +namespace clangd {
> +
> +struct SymbolLocation {
> + // The absolute path of the source file where a symbol occurs.
> + std::string FilePath;
> + // The 0-based offset to the first character of the symbol from the beginning
> + // of the source file.
> + unsigned StartOffset;
> + // The 0-based offset to the last character of the symbol from the beginning
> + // of the source file.
> + unsigned EndOffset;
> +};
> +
> +// The class identifies a particular C++ symbol (class, function, method, etc).
> +//
> +// As USRs (Unified Symbol Resolution) could be large, especially for functions
> +// with long type arguments, SymbolID is using 160-bits SHA1(USR) values to
> +// guarantee the uniqueness of symbols while using a relatively small amount of
> +// memory (vs storing USRs directly).
> +//
> +// SymbolID can be used as key in the symbol indexes to lookup the symbol.
> +class SymbolID {
> +public:
> + SymbolID() = default;
> + SymbolID(llvm::StringRef USR);
> +
> + bool operator==(const SymbolID& Sym) const {
> + return HashValue == Sym.HashValue;
> + }
> +
> +private:
> + friend class llvm::DenseMapInfo<clang::clangd::SymbolID>;
> +
> + std::array<uint8_t, 20> HashValue;
> +};
> +
> +// The class presents a C++ symbol, e.g. class, function.
> +//
> +// FIXME: instead of having own copy fields for each symbol, we can share
> +// storage from SymbolSlab.
> +struct Symbol {
> + // The ID of the symbol.
> + SymbolID ID;
> + // The qualified name of the symbol, e.g. Foo::bar.
> + std::string QualifiedName;
> + // The symbol information, like symbol kind.
> + index::SymbolInfo SymInfo;
> + // The location of the canonical declaration of the symbol.
> + //
> + // A C++ symbol could have multiple declarations and one definition (e.g.
> + // a function is declared in ".h" file, and is defined in ".cc" file).
> + // * For classes, the canonical declaration is usually definition.
> + // * For non-inline functions, the canonical declaration is a declaration
> + // (not a definition), which is usually declared in ".h" file.
> + SymbolLocation CanonicalDeclaration;
> +
> + // FIXME: add definition location of the symbol.
> + // FIXME: add all occurrences support.
> + // FIXME: add extra fields for index scoring signals.
> + // FIXME: add code completion information.
> +};
> +
> +// A symbol container that stores a set of symbols. The container will maintain
> +// the lifetime of the symbols.
> +//
> +// FIXME: Use a space-efficient implementation, a lot of Symbol fields could
> +// share the same storage.
> +class SymbolSlab {
> + public:
> + using const_iterator = llvm::DenseMap<SymbolID, Symbol>::const_iterator;
> +
> + SymbolSlab() = default;
> +
> + const_iterator begin() const;
> + const_iterator end() const;
> + const_iterator find(const SymbolID& SymID) const;
> +
> + // Once called, no more symbols would be added to the SymbolSlab. This
> + // operation is irreversible.
> + void freeze();
> +
> + void insert(Symbol S);
> +
> + private:
> + bool Frozen = false;
> +
> + llvm::DenseMap<SymbolID, Symbol> Symbols;
> +};
> +
> +} // namespace clangd
> +} // namespace clang
> +
> +namespace llvm {
> +
> +template <> struct DenseMapInfo<clang::clangd::SymbolID> {
> + static inline clang::clangd::SymbolID getEmptyKey() {
> + static clang::clangd::SymbolID EmptyKey("EMPTYKEY");
> + return EmptyKey;
> + }
> + static inline clang::clangd::SymbolID getTombstoneKey() {
> + static clang::clangd::SymbolID TombstoneKey("TOMBSTONEKEY");
> + return TombstoneKey;
> + }
> + static unsigned getHashValue(const clang::clangd::SymbolID &Sym) {
> + return hash_value(
> + ArrayRef<uint8_t>(Sym.HashValue.data(), Sym.HashValue.size()));
> + }
> + static bool isEqual(const clang::clangd::SymbolID &LHS,
> + const clang::clangd::SymbolID &RHS) {
> + return LHS == RHS;
> + }
> +};
> +
> +} // namespace llvm
> +
> +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
>
> Added: clang-tools-extra/trunk/clangd/index/SymbolCollector.cpp
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolCollector.cpp?rev=320486&view=auto
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/SymbolCollector.cpp (added)
> +++ clang-tools-extra/trunk/clangd/index/SymbolCollector.cpp Tue Dec 12 07:42:10 2017
> @@ -0,0 +1,102 @@
> +//===--- SymbolCollector.cpp -------------------------------------*- C++-*-===//
> +//
> +// The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +
> +#include "SymbolCollector.h"
> +
> +#include "clang/AST/ASTContext.h"
> +#include "clang/AST/Decl.h"
> +#include "clang/AST/DeclCXX.h"
> +#include "clang/Basic/SourceManager.h"
> +#include "clang/Index/IndexSymbol.h"
> +#include "clang/Index/USRGeneration.h"
> +#include "llvm/Support/MemoryBuffer.h"
> +#include "llvm/Support/Path.h"
> +
> +namespace clang {
> +namespace clangd {
> +
> +namespace {
> +// Make the Path absolute using the current working directory of the given
> +// SourceManager if the Path is not an absolute path.
> +//
> +// The Path can be a path relative to the build directory, or retrieved from
> +// the SourceManager.
> +std::string makeAbsolutePath(const SourceManager &SM, StringRef Path) {
> + llvm::SmallString<128> AbsolutePath(Path);
> + if (std::error_code EC =
> + SM.getFileManager().getVirtualFileSystem()->makeAbsolute(
> + AbsolutePath))
> + llvm::errs() << "Warning: could not make absolute file: '" << EC.message()
> + << '\n';
> + // Handle the symbolic link path case where the current working directory
> + // (getCurrentWorkingDirectory) is a symlink./ We always want to the real
> + // file path (instead of the symlink path) for the C++ symbols.
> + //
> + // Consider the following example:
> + //
> + // src dir: /project/src/foo.h
> + // current working directory (symlink): /tmp/build -> /project/src/
> + //
> + // The file path of Symbol is "/project/src/foo.h" instead of
> + // "/tmp/build/foo.h"
> + const DirectoryEntry *Dir = SM.getFileManager().getDirectory(
> + llvm::sys::path::parent_path(AbsolutePath.str()));
> + if (Dir) {
> + StringRef DirName = SM.getFileManager().getCanonicalName(Dir);
> + SmallVector<char, 128> AbsoluteFilename;
> + llvm::sys::path::append(AbsoluteFilename, DirName,
> + llvm::sys::path::filename(AbsolutePath.str()));
> + return llvm::StringRef(AbsoluteFilename.data(), AbsoluteFilename.size())
> + .str();
> + }
> + return AbsolutePath.str();
> +}
> +} // namespace
> +
> +// Always return true to continue indexing.
> +bool SymbolCollector::handleDeclOccurence(
> + const Decl *D, index::SymbolRoleSet Roles,
> + ArrayRef<index::SymbolRelation> Relations, FileID FID, unsigned Offset,
> + index::IndexDataConsumer::ASTNodeInfo ASTNode) {
> + // FIXME: collect all symbol references.
> + if (!(Roles & static_cast<unsigned>(index::SymbolRole::Declaration) ||
> + Roles & static_cast<unsigned>(index::SymbolRole::Definition)))
> + return true;
> +
> + if (const NamedDecl *ND = llvm::dyn_cast<NamedDecl>(D)) {
> + // FIXME: Should we include the internal linkage symbols?
> + if (!ND->hasExternalFormalLinkage() || ND->isInAnonymousNamespace())
> + return true;
> +
> + llvm::SmallVector<char, 128> Buff;
> + if (index::generateUSRForDecl(ND, Buff))
> + return true;
> +
> + std::string USR(Buff.data(), Buff.size());
> + auto ID = SymbolID(USR);
> + if (Symbols.find(ID) != Symbols.end())
> + return true;
> +
> + auto &SM = ND->getASTContext().getSourceManager();
> + SymbolLocation Location = {
> + makeAbsolutePath(SM, SM.getFilename(D->getLocation())),
> + SM.getFileOffset(D->getLocStart()), SM.getFileOffset(D->getLocEnd())};
> + Symbols.insert({std::move(ID), ND->getQualifiedNameAsString(),
> + index::getSymbolInfo(D), std::move(Location)});
> + }
> +
> + return true;
> +}
> +
> +void SymbolCollector::finish() {
> + Symbols.freeze();
> +}
> +
> +} // namespace clangd
> +} // namespace clang
>
> Added: clang-tools-extra/trunk/clangd/index/SymbolCollector.h
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolCollector.h?rev=320486&view=auto
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/SymbolCollector.h (added)
> +++ clang-tools-extra/trunk/clangd/index/SymbolCollector.h Tue Dec 12 07:42:10 2017
> @@ -0,0 +1,43 @@
> +//===--- SymbolCollector.h ---------------------------------------*- C++-*-===//
> +//
> +// The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +
> +#include "Index.h"
> +
> +#include "clang/Index/IndexDataConsumer.h"
> +#include "clang/Index/IndexSymbol.h"
> +
> +namespace clang {
> +namespace clangd {
> +
> +// Collect all symbols from an AST.
> +//
> +// Clients (e.g. clangd) can use SymbolCollector together with
> +// index::indexTopLevelDecls to retrieve all symbols when the source file is
> +// changed.
> +class SymbolCollector : public index::IndexDataConsumer {
> +public:
> + SymbolCollector() = default;
> +
> + bool
> + handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles,
> + ArrayRef<index::SymbolRelation> Relations, FileID FID,
> + unsigned Offset,
> + index::IndexDataConsumer::ASTNodeInfo ASTNode) override;
> +
> + void finish() override;
> +
> + SymbolSlab takeSymbols() const { return std::move(Symbols); }
> +
> +private:
> + // All Symbols collected from the AST.
> + SymbolSlab Symbols;
> +};
> +
> +} // namespace clangd
> +} // namespace clang
>
> Modified: clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt?rev=320486&r1=320485&r2=320486&view=diff
> ==============================================================================
> --- clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt (original)
> +++ clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt Tue Dec 12 07:42:10 2017
> @@ -16,6 +16,7 @@ add_extra_unittest(ClangdTests
> JSONExprTests.cpp
> TestFS.cpp
> TraceTests.cpp
> + SymbolCollectorTests.cpp
> )
>
> target_link_libraries(ClangdTests
>
> Added: clang-tools-extra/trunk/unittests/clangd/SymbolCollectorTests.cpp
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/SymbolCollectorTests.cpp?rev=320486&view=auto
> ==============================================================================
> --- clang-tools-extra/trunk/unittests/clangd/SymbolCollectorTests.cpp (added)
> +++ clang-tools-extra/trunk/unittests/clangd/SymbolCollectorTests.cpp Tue Dec 12 07:42:10 2017
> @@ -0,0 +1,110 @@
> +//===-- SymbolCollectorTests.cpp -------------------------------*- C++ -*-===//
> +//
> +// The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +
> +#include "index/SymbolCollector.h"
> +#include "clang/Index/IndexingAction.h"
> +#include "clang/Basic/FileManager.h"
> +#include "clang/Basic/FileSystemOptions.h"
> +#include "clang/Basic/VirtualFileSystem.h"
> +#include "clang/Frontend/CompilerInstance.h"
> +#include "clang/Tooling/Tooling.h"
> +#include "llvm/ADT/IntrusiveRefCntPtr.h"
> +#include "llvm/ADT/STLExtras.h"
> +#include "llvm/ADT/StringRef.h"
> +#include "llvm/Support/MemoryBuffer.h"
> +#include "gtest/gtest.h"
> +#include "gmock/gmock.h"
> +
> +#include <memory>
> +#include <string>
> +
> +using testing::UnorderedElementsAre;
> +using testing::Eq;
> +using testing::Field;
> +
> +// GMock helpers for matching Symbol.
> +MATCHER_P(QName, Name, "") { return arg.second.QualifiedName == Name; }
> +
> +namespace clang {
> +namespace clangd {
> +
> +namespace {
> +class SymbolIndexActionFactory : public tooling::FrontendActionFactory {
> + public:
> + SymbolIndexActionFactory() = default;
> +
> + clang::FrontendAction *create() override {
> + index::IndexingOptions IndexOpts;
> + IndexOpts.SystemSymbolFilter =
> + index::IndexingOptions::SystemSymbolFilterKind::All;
> + IndexOpts.IndexFunctionLocals = false;
> + Collector = std::make_shared<SymbolCollector>();
> + FrontendAction *Action =
> + index::createIndexingAction(Collector, IndexOpts, nullptr).release();
> + return Action;
> + }
> +
> + std::shared_ptr<SymbolCollector> Collector;
> +};
> +
> +class SymbolCollectorTest : public ::testing::Test {
> +public:
> + bool runSymbolCollector(StringRef HeaderCode, StringRef MainCode) {
> + llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem(
> + new vfs::InMemoryFileSystem);
> + llvm::IntrusiveRefCntPtr<FileManager> Files(
> + new FileManager(FileSystemOptions(), InMemoryFileSystem));
> +
> + const std::string FileName = "symbol.cc";
> + const std::string HeaderName = "symbols.h";
> + auto Factory = llvm::make_unique<SymbolIndexActionFactory>();
> +
> + tooling::ToolInvocation Invocation(
> + {"symbol_collector", "-fsyntax-only", "-std=c++11", FileName},
> + Factory->create(), Files.get(),
> + std::make_shared<PCHContainerOperations>());
> +
> + InMemoryFileSystem->addFile(HeaderName, 0,
> + llvm::MemoryBuffer::getMemBuffer(HeaderCode));
> +
> + std::string Content = "#include\"" + std::string(HeaderName) + "\"";
> + Content += "\n" + MainCode.str();
> + InMemoryFileSystem->addFile(FileName, 0,
> + llvm::MemoryBuffer::getMemBuffer(Content));
> + Invocation.run();
> + Symbols = Factory->Collector->takeSymbols();
> + return true;
> + }
> +
> +protected:
> + SymbolSlab Symbols;
> +};
> +
> +TEST_F(SymbolCollectorTest, CollectSymbol) {
> + const std::string Header = R"(
> + class Foo {
> + void f();
> + };
> + void f1();
> + inline void f2() {}
> + )";
> + const std::string Main = R"(
> + namespace {
> + void ff() {} // ignore
> + }
> + void f1() {}
> + )";
> + runSymbolCollector(Header, Main);
> + EXPECT_THAT(Symbols, UnorderedElementsAre(QName("Foo"), QName("Foo::f"),
> + QName("f1"), QName("f2")));
> +}
> +
> +} // namespace
> +} // namespace clangd
> +} // namespace clang
>
>
> _______________________________________________
> cfe-commits mailing list
> cfe-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
More information about the cfe-commits
mailing list