[clang-tools-extra] r321092 - [clangd] Build dynamic index and use it for code completion.
Eric Liu via cfe-commits
cfe-commits at lists.llvm.org
Tue Dec 19 10:00:37 PST 2017
Author: ioeric
Date: Tue Dec 19 10:00:37 2017
New Revision: 321092
URL: http://llvm.org/viewvc/llvm-project?rev=321092&view=rev
Log:
[clangd] Build dynamic index and use it for code completion.
Reviewers: sammccall
Reviewed By: sammccall
Subscribers: klimek, ilya-biryukov, cfe-commits
Differential Revision: https://reviews.llvm.org/D41289
Modified:
clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
clang-tools-extra/trunk/clangd/ClangdLSPServer.h
clang-tools-extra/trunk/clangd/ClangdServer.cpp
clang-tools-extra/trunk/clangd/ClangdServer.h
clang-tools-extra/trunk/clangd/ClangdUnit.cpp
clang-tools-extra/trunk/clangd/ClangdUnit.h
clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp
clang-tools-extra/trunk/clangd/ClangdUnitStore.h
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp
Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp Tue Dec 19 10:00:37 2017
@@ -282,10 +282,11 @@ ClangdLSPServer::ClangdLSPServer(JSONOut
bool StorePreamblesInMemory,
const clangd::CodeCompleteOptions &CCOpts,
llvm::Optional<StringRef> ResourceDir,
- llvm::Optional<Path> CompileCommandsDir)
+ llvm::Optional<Path> CompileCommandsDir,
+ bool BuildDynamicSymbolIndex)
: Out(Out), CDB(std::move(CompileCommandsDir)), CCOpts(CCOpts),
Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount,
- StorePreamblesInMemory, ResourceDir) {}
+ StorePreamblesInMemory, BuildDynamicSymbolIndex, ResourceDir) {}
bool ClangdLSPServer::run(std::istream &In) {
assert(!IsDone && "Run was called before");
Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.h?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.h Tue Dec 19 10:00:37 2017
@@ -34,7 +34,8 @@ public:
bool StorePreamblesInMemory,
const clangd::CodeCompleteOptions &CCOpts,
llvm::Optional<StringRef> ResourceDir,
- llvm::Optional<Path> CompileCommandsDir);
+ llvm::Optional<Path> CompileCommandsDir,
+ bool BuildDynamicSymbolIndex);
/// Run LSP server loop, receiving input for it from \p In. \p In must be
/// opened in binary mode. Output will be written using Out variable passed to
Modified: clang-tools-extra/trunk/clangd/ClangdServer.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.cpp?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdServer.cpp Tue Dec 19 10:00:37 2017
@@ -134,8 +134,19 @@ ClangdServer::ClangdServer(GlobalCompila
FileSystemProvider &FSProvider,
unsigned AsyncThreadsCount,
bool StorePreamblesInMemory,
+ bool BuildDynamicSymbolIndex,
llvm::Optional<StringRef> ResourceDir)
: CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider),
+ FileIdx(BuildDynamicSymbolIndex ? new FileIndex() : nullptr),
+ // Pass a callback into `Units` to extract symbols from a newly parsed
+ // file and rebuild the file index synchronously each time an AST is
+ // parsed.
+ // FIXME(ioeric): this can be slow and we may be able to index on less
+ // critical paths.
+ Units(FileIdx
+ ? [this](const Context &Ctx, PathRef Path,
+ ParsedAST *AST) { FileIdx->update(Ctx, Path, AST); }
+ : ASTParsedCallback()),
ResourceDir(ResourceDir ? ResourceDir->str() : getStandardResourceDir()),
PCHs(std::make_shared<PCHContainerOperations>()),
StorePreamblesInMemory(StorePreamblesInMemory),
@@ -238,6 +249,8 @@ void ClangdServer::codeComplete(
Resources->getPossiblyStalePreamble();
// Copy completion options for passing them to async task handler.
auto CodeCompleteOpts = Opts;
+ if (FileIdx)
+ CodeCompleteOpts.Index = FileIdx.get();
// A task that will be run asynchronously.
auto Task =
// 'mutable' to reassign Preamble variable.
Modified: clang-tools-extra/trunk/clangd/ClangdServer.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.h?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdServer.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdServer.h Tue Dec 19 10:00:37 2017
@@ -17,6 +17,7 @@
#include "Function.h"
#include "GlobalCompilationDatabase.h"
#include "Protocol.h"
+#include "index/FileIndex.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "clang/Tooling/Core/Replacement.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -195,10 +196,15 @@ public:
///
/// \p StorePreamblesInMemory defines whether the Preambles generated by
/// clangd are stored in-memory or on disk.
+ ///
+ /// If \p BuildDynamicSymbolIndex is true, ClangdServer builds a dynamic
+ /// in-memory index for symbols in all opened files and uses the index to
+ /// augment code completion results.
ClangdServer(GlobalCompilationDatabase &CDB,
DiagnosticsConsumer &DiagConsumer,
FileSystemProvider &FSProvider, unsigned AsyncThreadsCount,
bool StorePreamblesInMemory,
+ bool BuildDynamicSymbolIndex = false,
llvm::Optional<StringRef> ResourceDir = llvm::None);
/// Set the root path of the workspace.
@@ -330,6 +336,8 @@ private:
DiagnosticsConsumer &DiagConsumer;
FileSystemProvider &FSProvider;
DraftStore DraftMgr;
+ /// If set, this manages index for symbols in opened files.
+ std::unique_ptr<FileIndex> FileIdx;
CppFileCollection Units;
std::string ResourceDir;
// If set, this represents the workspace path.
Modified: clang-tools-extra/trunk/clangd/ClangdUnit.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.cpp?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnit.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp Tue Dec 19 10:00:37 2017
@@ -357,17 +357,21 @@ ParsedASTWrapper::ParsedASTWrapper(llvm:
std::shared_ptr<CppFile>
CppFile::Create(PathRef FileName, tooling::CompileCommand Command,
bool StorePreamblesInMemory,
- std::shared_ptr<PCHContainerOperations> PCHs) {
- return std::shared_ptr<CppFile>(new CppFile(
- FileName, std::move(Command), StorePreamblesInMemory, std::move(PCHs)));
+ std::shared_ptr<PCHContainerOperations> PCHs,
+ ASTParsedCallback ASTCallback) {
+ return std::shared_ptr<CppFile>(
+ new CppFile(FileName, std::move(Command), StorePreamblesInMemory,
+ std::move(PCHs), std::move(ASTCallback)));
}
CppFile::CppFile(PathRef FileName, tooling::CompileCommand Command,
bool StorePreamblesInMemory,
- std::shared_ptr<PCHContainerOperations> PCHs)
+ std::shared_ptr<PCHContainerOperations> PCHs,
+ ASTParsedCallback ASTCallback)
: FileName(FileName), Command(std::move(Command)),
StorePreamblesInMemory(StorePreamblesInMemory), RebuildCounter(0),
- RebuildInProgress(false), PCHs(std::move(PCHs)) {
+ RebuildInProgress(false), PCHs(std::move(PCHs)),
+ ASTCallback(std::move(ASTCallback)) {
// FIXME(ibiryukov): we should pass a proper Context here.
log(Context::empty(), "Opened file " + FileName + " with command [" +
this->Command.Directory + "] " +
@@ -570,6 +574,8 @@ CppFile::deferRebuild(StringRef NewConte
if (NewAST) {
Diagnostics.insert(Diagnostics.end(), NewAST->getDiagnostics().begin(),
NewAST->getDiagnostics().end());
+ if (That->ASTCallback)
+ That->ASTCallback(Ctx, That->FileName, NewAST.getPointer());
} else {
// Don't report even Preamble diagnostics if we coulnd't build AST.
Diagnostics.clear();
Modified: clang-tools-extra/trunk/clangd/ClangdUnit.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.h?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnit.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnit.h Tue Dec 19 10:00:37 2017
@@ -136,6 +136,9 @@ private:
mutable llvm::Optional<ParsedAST> AST;
};
+using ASTParsedCallback =
+ std::function<void(const Context &Ctx, PathRef Path, ParsedAST *)>;
+
/// Manages resources, required by clangd. Allows to rebuild file with new
/// contents, and provides AST and Preamble for it.
class CppFile : public std::enable_shared_from_this<CppFile> {
@@ -145,12 +148,14 @@ public:
static std::shared_ptr<CppFile>
Create(PathRef FileName, tooling::CompileCommand Command,
bool StorePreamblesInMemory,
- std::shared_ptr<PCHContainerOperations> PCHs);
+ std::shared_ptr<PCHContainerOperations> PCHs,
+ ASTParsedCallback ASTCallback);
private:
CppFile(PathRef FileName, tooling::CompileCommand Command,
bool StorePreamblesInMemory,
- std::shared_ptr<PCHContainerOperations> PCHs);
+ std::shared_ptr<PCHContainerOperations> PCHs,
+ ASTParsedCallback ASTCallback);
public:
CppFile(CppFile const &) = delete;
@@ -252,6 +257,9 @@ private:
std::shared_ptr<const PreambleData> LatestAvailablePreamble;
/// Utility class, required by clang.
std::shared_ptr<PCHContainerOperations> PCHs;
+ /// This is called after the file is parsed. This can be nullptr if there is
+ /// no callback.
+ ASTParsedCallback ASTCallback;
};
/// Get the beginning SourceLocation at a specified \p Pos.
Modified: clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp Tue Dec 19 10:00:37 2017
@@ -41,13 +41,14 @@ CppFileCollection::recreateFileIfCompile
It = OpenedFiles
.try_emplace(File, CppFile::Create(File, std::move(NewCommand),
StorePreamblesInMemory,
- std::move(PCHs)))
+ std::move(PCHs), ASTCallback))
.first;
} else if (!compileCommandsAreEqual(It->second->getCompileCommand(),
NewCommand)) {
Result.RemovedFile = std::move(It->second);
- It->second = CppFile::Create(File, std::move(NewCommand),
- StorePreamblesInMemory, std::move(PCHs));
+ It->second =
+ CppFile::Create(File, std::move(NewCommand), StorePreamblesInMemory,
+ std::move(PCHs), ASTCallback);
}
Result.FileInCollection = It->second;
return Result;
Modified: clang-tools-extra/trunk/clangd/ClangdUnitStore.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnitStore.h?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnitStore.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnitStore.h Tue Dec 19 10:00:37 2017
@@ -25,6 +25,11 @@ class Logger;
/// Thread-safe mapping from FileNames to CppFile.
class CppFileCollection {
public:
+ /// \p ASTCallback is called when a file is parsed synchronously. This should
+ /// not be expensive since it blocks diagnostics.
+ explicit CppFileCollection(ASTParsedCallback ASTCallback)
+ : ASTCallback(std::move(ASTCallback)) {}
+
std::shared_ptr<CppFile>
getOrCreateFile(PathRef File, PathRef ResourceDir,
GlobalCompilationDatabase &CDB, bool StorePreamblesInMemory,
@@ -38,7 +43,7 @@ public:
It = OpenedFiles
.try_emplace(File, CppFile::Create(File, std::move(Command),
StorePreamblesInMemory,
- std::move(PCHs)))
+ std::move(PCHs), ASTCallback))
.first;
}
return It->second;
@@ -85,6 +90,7 @@ private:
std::mutex Mutex;
llvm::StringMap<std::shared_ptr<CppFile>> OpenedFiles;
+ ASTParsedCallback ASTCallback;
};
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Tue Dec 19 10:00:37 2017
@@ -90,6 +90,13 @@ static llvm::cl::opt<Path> TraceFile(
"Trace internal events and timestamps in chrome://tracing JSON format"),
llvm::cl::init(""), llvm::cl::Hidden);
+static llvm::cl::opt<bool> EnableIndexBasedCompletion(
+ "enable-index-based-completion",
+ llvm::cl::desc(
+ "Enable index-based global code completion (experimental). Clangd will "
+ "use index built from symbols in opened files"),
+ llvm::cl::init(false), llvm::cl::Hidden);
+
int main(int argc, char *argv[]) {
llvm::cl::ParseCommandLineOptions(argc, argv, "clangd");
@@ -180,7 +187,8 @@ int main(int argc, char *argv[]) {
CCOpts.IncludeIneligibleResults = IncludeIneligibleResults;
// Initialize and run ClangdLSPServer.
ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory,
- CCOpts, ResourceDirRef, CompileCommandsDirPath);
+ CCOpts, ResourceDirRef, CompileCommandsDirPath,
+ EnableIndexBasedCompletion);
constexpr int NoShutdownRequestErrorCode = 1;
llvm::set_thread_name("clangd.main");
return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode;
Modified: clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp?rev=321092&r1=321091&r2=321092&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp Tue Dec 19 10:00:37 2017
@@ -558,6 +558,38 @@ TEST(CompletionTest, FullyQualifiedScope
EXPECT_THAT(Results.items, Has("XYZ", CompletionItemKind::Class));
}
+TEST(CompletionTest, ASTIndexMultiFile) {
+ MockFSProvider FS;
+ MockCompilationDatabase CDB;
+ IgnoreDiagnostics DiagConsumer;
+ ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
+ /*StorePreamblesInMemory=*/true,
+ /*BuildDynamicSymbolIndex=*/true);
+
+ Server
+ .addDocument(Context::empty(), getVirtualTestFilePath("foo.cpp"), R"cpp(
+ namespace ns { class XYZ {}; void foo() {} }
+ )cpp")
+ .wait();
+
+ auto File = getVirtualTestFilePath("bar.cpp");
+ auto Test = parseTextMarker(R"cpp(
+ namespace ns { class XXX {}; void fooooo() {} }
+ void f() { ns::^ }
+ )cpp");
+ Server.addDocument(Context::empty(), File, Test.Text).wait();
+
+ auto Results = Server.codeComplete(Context::empty(), File, Test.MarkerPos, {})
+ .get()
+ .second.Value;
+ // "XYZ" and "foo" are not included in the file being completed but are still
+ // visible through the index.
+ EXPECT_THAT(Results.items, Has("XYZ", CompletionItemKind::Class));
+ EXPECT_THAT(Results.items, Has("foo", CompletionItemKind::Function));
+ EXPECT_THAT(Results.items, Has("XXX", CompletionItemKind::Class));
+ EXPECT_THAT(Results.items, Has("fooooo", CompletionItemKind::Function));
+}
+
} // namespace
} // namespace clangd
} // namespace clang
More information about the cfe-commits
mailing list