[clang-tools-extra] r322191 - [clangd] Add static index for the global code completion.
Haojian Wu via cfe-commits
cfe-commits at lists.llvm.org
Wed Jan 10 06:44:34 PST 2018
Author: hokein
Date: Wed Jan 10 06:44:34 2018
New Revision: 322191
URL: http://llvm.org/viewvc/llvm-project?rev=322191&view=rev
Log:
[clangd] Add static index for the global code completion.
Summary:
Use the YAML-format symbols (generated by the global-symbol-builder tool) to
do the global code completion.
It is **experimental** only , but it allows us to experience global code
completion on a relatively small project.
Tested with LLVM project.
Reviewers: sammccall, ioeric
Reviewed By: sammccall, ioeric
Subscribers: klimek, ilya-biryukov, cfe-commits
Differential Revision: https://reviews.llvm.org/D41668
Modified:
clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
clang-tools-extra/trunk/clangd/ClangdLSPServer.h
clang-tools-extra/trunk/clangd/ClangdServer.cpp
clang-tools-extra/trunk/clangd/ClangdServer.h
clang-tools-extra/trunk/clangd/CodeComplete.cpp
clang-tools-extra/trunk/clangd/CodeComplete.h
clang-tools-extra/trunk/clangd/index/MemIndex.cpp
clang-tools-extra/trunk/clangd/index/MemIndex.h
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp
Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp Wed Jan 10 06:44:34 2018
@@ -283,10 +283,12 @@ ClangdLSPServer::ClangdLSPServer(JSONOut
const clangd::CodeCompleteOptions &CCOpts,
llvm::Optional<StringRef> ResourceDir,
llvm::Optional<Path> CompileCommandsDir,
- bool BuildDynamicSymbolIndex)
+ bool BuildDynamicSymbolIndex,
+ SymbolIndex *StaticIdx)
: Out(Out), CDB(std::move(CompileCommandsDir)), CCOpts(CCOpts),
Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount,
- StorePreamblesInMemory, BuildDynamicSymbolIndex, ResourceDir) {}
+ StorePreamblesInMemory, BuildDynamicSymbolIndex, StaticIdx,
+ ResourceDir) {}
bool ClangdLSPServer::run(std::istream &In) {
assert(!IsDone && "Run was called before");
Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.h?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.h Wed Jan 10 06:44:34 2018
@@ -22,6 +22,7 @@ namespace clang {
namespace clangd {
class JSONOutput;
+class SymbolIndex;
/// This class provides implementation of an LSP server, glueing the JSON
/// dispatch and ClangdServer together.
@@ -35,7 +36,8 @@ public:
const clangd::CodeCompleteOptions &CCOpts,
llvm::Optional<StringRef> ResourceDir,
llvm::Optional<Path> CompileCommandsDir,
- bool BuildDynamicSymbolIndex);
+ bool BuildDynamicSymbolIndex,
+ SymbolIndex *StaticIdx = nullptr);
/// Run LSP server loop, receiving input for it from \p In. \p In must be
/// opened in binary mode. Output will be written using Out variable passed to
Modified: clang-tools-extra/trunk/clangd/ClangdServer.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.cpp?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdServer.cpp Wed Jan 10 06:44:34 2018
@@ -134,10 +134,11 @@ ClangdServer::ClangdServer(GlobalCompila
FileSystemProvider &FSProvider,
unsigned AsyncThreadsCount,
bool StorePreamblesInMemory,
- bool BuildDynamicSymbolIndex,
+ bool BuildDynamicSymbolIndex, SymbolIndex *StaticIdx,
llvm::Optional<StringRef> ResourceDir)
: CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider),
FileIdx(BuildDynamicSymbolIndex ? new FileIndex() : nullptr),
+ StaticIdx(StaticIdx),
// Pass a callback into `Units` to extract symbols from a newly parsed
// file and rebuild the file index synchronously each time an AST is
// parsed.
@@ -251,6 +252,8 @@ void ClangdServer::codeComplete(
auto CodeCompleteOpts = Opts;
if (FileIdx)
CodeCompleteOpts.Index = FileIdx.get();
+ if (StaticIdx)
+ CodeCompleteOpts.StaticIndex = StaticIdx;
// Copy File, as it is a PathRef that will go out of scope before Task is
// executed.
Modified: clang-tools-extra/trunk/clangd/ClangdServer.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.h?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdServer.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdServer.h Wed Jan 10 06:44:34 2018
@@ -200,11 +200,15 @@ public:
/// If \p BuildDynamicSymbolIndex is true, ClangdServer builds a dynamic
/// in-memory index for symbols in all opened files and uses the index to
/// augment code completion results.
+ ///
+ /// If \p StaticIdx is set, ClangdServer uses the index for global code
+ /// completion.
ClangdServer(GlobalCompilationDatabase &CDB,
DiagnosticsConsumer &DiagConsumer,
FileSystemProvider &FSProvider, unsigned AsyncThreadsCount,
bool StorePreamblesInMemory,
bool BuildDynamicSymbolIndex = false,
+ SymbolIndex *StaticIdx = nullptr,
llvm::Optional<StringRef> ResourceDir = llvm::None);
/// Set the root path of the workspace.
@@ -338,6 +342,11 @@ private:
DraftStore DraftMgr;
/// If set, this manages index for symbols in opened files.
std::unique_ptr<FileIndex> FileIdx;
+ /// If set, this provides static index for project-wide global symbols.
+ /// clangd global code completion result will come from the static index and
+ /// the `FileIdx` above.
+ /// No owned, the life time is managed by clangd embedders.
+ SymbolIndex *StaticIdx;
CppFileCollection Units;
std::string ResourceDir;
// If set, this represents the workspace path.
Modified: clang-tools-extra/trunk/clangd/CodeComplete.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CodeComplete.cpp?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/CodeComplete.cpp (original)
+++ clang-tools-extra/trunk/clangd/CodeComplete.cpp Wed Jan 10 06:44:34 2018
@@ -23,6 +23,7 @@
#include "clang/Frontend/FrontendActions.h"
#include "clang/Sema/CodeCompleteConsumer.h"
#include "clang/Sema/Sema.h"
+#include "llvm/Support/Format.h"
#include <queue>
namespace clang {
@@ -558,10 +559,27 @@ bool invokeCodeComplete(const Context &C
}
CompletionItem indexCompletionItem(const Symbol &Sym, llvm::StringRef Filter,
- const SpecifiedScope &SSInfo) {
+ const SpecifiedScope &SSInfo,
+ llvm::StringRef DebuggingLabel = "") {
CompletionItem Item;
Item.kind = toCompletionItemKind(Sym.SymInfo.Kind);
- Item.label = Sym.Name;
+ // Add DebuggingLabel to the completion results if DebuggingLabel is not
+ // empty.
+ //
+ // For symbols from static index, there are prefix "[G]" in the
+ // results (which is used for debugging purpose).
+ // So completion list will be like:
+ // clang::symbol_from_dynamic_index
+ // [G]clang::symbol_from_static_index
+ //
+ // FIXME: Find out a better way to show the index source.
+ if (!DebuggingLabel.empty()) {
+ llvm::raw_string_ostream Label(Item.label);
+ Label << llvm::format("[%s]%s", DebuggingLabel.str().c_str(),
+ Sym.Name.str().c_str());
+ } else {
+ Item.label = Sym.Name;
+ }
// FIXME(ioeric): support inserting/replacing scope qualifiers.
// FIXME(ioeric): support snippets.
@@ -582,7 +600,8 @@ CompletionItem indexCompletionItem(const
void completeWithIndex(const Context &Ctx, const SymbolIndex &Index,
llvm::StringRef Code, const SpecifiedScope &SSInfo,
- llvm::StringRef Filter, CompletionList *Items) {
+ llvm::StringRef Filter, CompletionList *Items,
+ llvm::StringRef DebuggingLabel = "") {
FuzzyFindRequest Req;
Req.Query = Filter;
// FIXME(ioeric): add more possible scopes based on using namespaces and
@@ -590,8 +609,9 @@ void completeWithIndex(const Context &Ct
StringRef Scope = SSInfo.Resolved.empty() ? SSInfo.Written : SSInfo.Resolved;
Req.Scopes = {Scope.trim(':').str()};
- Items->isIncomplete = !Index.fuzzyFind(Ctx, Req, [&](const Symbol &Sym) {
- Items->items.push_back(indexCompletionItem(Sym, Filter, SSInfo));
+ Items->isIncomplete |= !Index.fuzzyFind(Ctx, Req, [&](const Symbol &Sym) {
+ Items->items.push_back(
+ indexCompletionItem(Sym, Filter, SSInfo, DebuggingLabel));
});
}
@@ -644,13 +664,19 @@ CompletionList codeComplete(const Contex
invokeCodeComplete(Ctx, std::move(Consumer), Opts.getClangCompleteOpts(),
FileName, Command, Preamble, Contents, Pos, std::move(VFS),
std::move(PCHs));
- if (Opts.Index && CompletedName.SSInfo) {
- if (!Results.items.empty())
- log(Ctx, "WARNING: Got completion results from sema for completion on "
- "qualified ID while symbol index is provided.");
- Results.items.clear();
- completeWithIndex(Ctx, *Opts.Index, Contents, *CompletedName.SSInfo,
- CompletedName.Filter, &Results);
+
+ // Got scope specifier (ns::f^) for code completion from sema, try to query
+ // global symbols from indexes.
+ if (CompletedName.SSInfo) {
+ // FIXME: figure out a good algorithm to merge symbols from different
+ // sources (dynamic index, static index, AST symbols from clang's completion
+ // engine).
+ if (Opts.Index)
+ completeWithIndex(Ctx, *Opts.Index, Contents, *CompletedName.SSInfo,
+ CompletedName.Filter, &Results);
+ if (Opts.StaticIndex)
+ completeWithIndex(Ctx, *Opts.StaticIndex, Contents, *CompletedName.SSInfo,
+ CompletedName.Filter, &Results, /*DebuggingLabel=*/"G");
}
return Results;
}
Modified: clang-tools-extra/trunk/clangd/CodeComplete.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CodeComplete.h?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/CodeComplete.h (original)
+++ clang-tools-extra/trunk/clangd/CodeComplete.h Wed Jan 10 06:44:34 2018
@@ -67,6 +67,10 @@ struct CodeCompleteOptions {
/// FIXME(ioeric): we might want a better way to pass the index around inside
/// clangd.
const SymbolIndex *Index = nullptr;
+
+ // Populated internally by clangd, do not set.
+ /// Static index for project-wide global symbols.
+ const SymbolIndex *StaticIndex = nullptr;
};
/// Get code completions at a specified \p Pos in \p FileName.
Modified: clang-tools-extra/trunk/clangd/index/MemIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.cpp?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/MemIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.cpp Wed Jan 10 06:44:34 2018
@@ -53,5 +53,21 @@ bool MemIndex::fuzzyFind(
return true;
}
+std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
+ struct Snapshot {
+ SymbolSlab Slab;
+ std::vector<const Symbol *> Pointers;
+ };
+ auto Snap = std::make_shared<Snapshot>();
+ Snap->Slab = std::move(Slab);
+ for (auto &Sym : Snap->Slab)
+ Snap->Pointers.push_back(&Sym);
+ auto S = std::shared_ptr<std::vector<const Symbol *>>(std::move(Snap),
+ &Snap->Pointers);
+ auto MemIdx = llvm::make_unique<MemIndex>();
+ MemIdx->build(std::move(S));
+ return std::move(MemIdx);
+}
+
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/index/MemIndex.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.h?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/MemIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.h Wed Jan 10 06:44:34 2018
@@ -24,6 +24,9 @@ public:
/// accessible as long as `Symbols` is kept alive.
void build(std::shared_ptr<std::vector<const Symbol *>> Symbols);
+ /// \brief Build index from a symbol slab.
+ static std::unique_ptr<SymbolIndex> build(SymbolSlab Slab);
+
bool
fuzzyFind(const Context &Ctx, const FuzzyFindRequest &Req,
llvm::function_ref<void(const Symbol &)> Callback) const override;
Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Wed Jan 10 06:44:34 2018
@@ -11,6 +11,7 @@
#include "JSONRPCDispatcher.h"
#include "Path.h"
#include "Trace.h"
+#include "index/SymbolYAML.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
@@ -26,7 +27,24 @@ using namespace clang::clangd;
namespace {
enum class PCHStorageFlag { Disk, Memory };
+
+// Build an in-memory static index for global symbols from a YAML-format file.
+// The size of global symbols should be relatively small, so that all symbols
+// can be managed in memory.
+std::unique_ptr<SymbolIndex> BuildStaticIndex(llvm::StringRef YamlSymbolFile) {
+ auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
+ if (!Buffer) {
+ llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
+ return nullptr;
+ }
+ auto Slab = SymbolFromYAML(Buffer.get()->getBuffer());
+ SymbolSlab::Builder SymsBuilder;
+ for (auto Sym : Slab)
+ SymsBuilder.insert(Sym);
+
+ return MemIndex::build(std::move(SymsBuilder).build());
}
+} // namespace
static llvm::cl::opt<Path> CompileCommandsDir(
"compile-commands-dir",
@@ -97,6 +115,15 @@ static llvm::cl::opt<bool> EnableIndexBa
"use index built from symbols in opened files"),
llvm::cl::init(false), llvm::cl::Hidden);
+static llvm::cl::opt<Path> YamlSymbolFile(
+ "yaml-symbol-file",
+ llvm::cl::desc(
+ "YAML-format global symbol file to build the static index. Clangd will "
+ "use the static index for global code completion.\n"
+ "WARNING: This option is experimental only, and will be removed "
+ "eventually. Don't rely on it."),
+ llvm::cl::init(""), llvm::cl::Hidden);
+
int main(int argc, char *argv[]) {
llvm::cl::ParseCommandLineOptions(argc, argv, "clangd");
@@ -182,13 +209,16 @@ int main(int argc, char *argv[]) {
// Change stdin to binary to not lose \r\n on windows.
llvm::sys::ChangeStdinToBinary();
+ std::unique_ptr<SymbolIndex> StaticIdx;
+ if (EnableIndexBasedCompletion && !YamlSymbolFile.empty())
+ StaticIdx = BuildStaticIndex(YamlSymbolFile);
clangd::CodeCompleteOptions CCOpts;
CCOpts.EnableSnippets = EnableSnippets;
CCOpts.IncludeIneligibleResults = IncludeIneligibleResults;
// Initialize and run ClangdLSPServer.
ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory,
CCOpts, ResourceDirRef, CompileCommandsDirPath,
- EnableIndexBasedCompletion);
+ EnableIndexBasedCompletion, StaticIdx.get());
constexpr int NoShutdownRequestErrorCode = 1;
llvm::set_thread_name("clangd.main");
return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode;
Modified: clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp?rev=322191&r1=322190&r2=322191&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp Wed Jan 10 06:44:34 2018
@@ -455,12 +455,6 @@ TEST(SignatureHelpTest, ActiveArg) {
std::unique_ptr<SymbolIndex> simpleIndexFromSymbols(
std::vector<std::pair<std::string, index::SymbolKind>> Symbols) {
- auto I = llvm::make_unique<MemIndex>();
- struct Snapshot {
- SymbolSlab Slab;
- std::vector<const Symbol *> Pointers;
- };
- auto Snap = std::make_shared<Snapshot>();
SymbolSlab::Builder Slab;
for (const auto &Pair : Symbols) {
Symbol Sym;
@@ -478,13 +472,7 @@ std::unique_ptr<SymbolIndex> simpleIndex
Sym.SymInfo.Kind = Pair.second;
Slab.insert(Sym);
}
- Snap->Slab = std::move(Slab).build();
- for (auto &Iter : Snap->Slab)
- Snap->Pointers.push_back(&Iter);
- auto S = std::shared_ptr<std::vector<const Symbol *>>(std::move(Snap),
- &Snap->Pointers);
- I->build(std::move(S));
- return std::move(I);
+ return MemIndex::build(std::move(Slab).build());
}
TEST(CompletionTest, NoIndex) {
@@ -499,6 +487,23 @@ TEST(CompletionTest, NoIndex) {
EXPECT_THAT(Results.items, Has("No"));
}
+TEST(CompletionTest, StaticAndDynamicIndex) {
+ clangd::CodeCompleteOptions Opts;
+ auto StaticIdx =
+ simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class}});
+ Opts.StaticIndex = StaticIdx.get();
+ auto DynamicIdx =
+ simpleIndexFromSymbols({{"ns::foo", index::SymbolKind::Function}});
+ Opts.Index = DynamicIdx.get();
+
+ auto Results = completions(R"cpp(
+ void f() { ::ns::^ }
+ )cpp",
+ Opts);
+ EXPECT_THAT(Results.items, Contains(Labeled("[G]XYZ")));
+ EXPECT_THAT(Results.items, Contains(Labeled("foo")));
+}
+
TEST(CompletionTest, SimpleIndexBased) {
clangd::CodeCompleteOptions Opts;
auto I = simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class},
More information about the cfe-commits
mailing list