[clang-tools-extra] r366458 - [clangd] Refactor background-index shard loading
Azhar Mohammed via cfe-commits
cfe-commits at lists.llvm.org
Thu Jul 18 18:21:22 PDT 2019
Hi Kadir
This change is causing test failures, can you please look into it. Refer to http://green.lab.llvm.org/green/job/clang-stage1-configure-RA/58104/testReport/.
Assertion failed: (TUsIt != FileToTU.end() && "No TU registered for the shard"), function takeResult, file /Users/buildslave/jenkins/workspace/clang-stage1-configure-RA/llvm/tools/clang/tools/extra/clangd/index/BackgroundIndexLoader.cpp, line 131.
Failing Tests (10):
Clangd :: did-change-configuration-params.test
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.CmdLineHash
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.DirectIncludesTest
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.IndexTwoFiles
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.NoCrashOnErrorFile
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.NoDotsInAbsPath
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.ShardStorageEmptyFile
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.ShardStorageLoad
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.ShardStorageTest
Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.UncompilableFiles
> On Jul 18, 2019, at 9:25 AM, Kadir Cetinkaya via cfe-commits <cfe-commits at lists.llvm.org> wrote:
>
> Author: kadircet
> Date: Thu Jul 18 09:25:36 2019
> New Revision: 366458
>
> URL: http://llvm.org/viewvc/llvm-project?rev=366458&view=rev
> Log:
> [clangd] Refactor background-index shard loading
>
> Reviewers: sammccall
>
> Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, cfe-commits
>
> Tags: #clang
>
> Differential Revision: https://reviews.llvm.org/D64712
>
> Added:
> clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp
> clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h
> Modified:
> clang-tools-extra/trunk/clangd/CMakeLists.txt
> clang-tools-extra/trunk/clangd/index/Background.cpp
> clang-tools-extra/trunk/clangd/index/Background.h
> clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp
> clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h
> clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp
>
> Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=366458&r1=366457&r2=366458&view=diff
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
> +++ clang-tools-extra/trunk/clangd/CMakeLists.txt Thu Jul 18 09:25:36 2019
> @@ -73,6 +73,7 @@ add_clang_library(clangDaemon
> XRefs.cpp
>
> index/Background.cpp
> + index/BackgroundIndexLoader.cpp
> index/BackgroundIndexStorage.cpp
> index/BackgroundQueue.cpp
> index/BackgroundRebuild.cpp
>
> Modified: clang-tools-extra/trunk/clangd/index/Background.cpp
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.cpp?rev=366458&r1=366457&r2=366458&view=diff
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/Background.cpp (original)
> +++ clang-tools-extra/trunk/clangd/index/Background.cpp Thu Jul 18 09:25:36 2019
> @@ -10,6 +10,7 @@
> #include "ClangdUnit.h"
> #include "Compiler.h"
> #include "Context.h"
> +#include "FSProvider.h"
> #include "Headers.h"
> #include "Logger.h"
> #include "Path.h"
> @@ -18,6 +19,7 @@
> #include "Threading.h"
> #include "Trace.h"
> #include "URI.h"
> +#include "index/BackgroundIndexLoader.h"
> #include "index/FileIndex.h"
> #include "index/IndexAction.h"
> #include "index/MemIndex.h"
> @@ -28,6 +30,8 @@
> #include "clang/Basic/SourceLocation.h"
> #include "clang/Basic/SourceManager.h"
> #include "clang/Driver/Types.h"
> +#include "llvm/ADT/ArrayRef.h"
> +#include "llvm/ADT/DenseSet.h"
> #include "llvm/ADT/Hashing.h"
> #include "llvm/ADT/STLExtras.h"
> #include "llvm/ADT/ScopeExit.h"
> @@ -42,6 +46,7 @@
> #include <atomic>
> #include <chrono>
> #include <condition_variable>
> +#include <cstddef>
> #include <memory>
> #include <mutex>
> #include <numeric>
> @@ -49,6 +54,8 @@
> #include <random>
> #include <string>
> #include <thread>
> +#include <utility>
> +#include <vector>
>
> namespace clang {
> namespace clangd {
> @@ -119,6 +126,18 @@ llvm::SmallString<128> getAbsolutePath(c
> }
> return AbsolutePath;
> }
> +
> +bool shardIsStale(const LoadedShard &LS, llvm::vfs::FileSystem *FS) {
> + auto Buf = FS->getBufferForFile(LS.AbsolutePath);
> + if (!Buf) {
> + elog("Background-index: Couldn't read {0} to validate stored index: {1}",
> + LS.AbsolutePath, Buf.getError().message());
> + // There is no point in indexing an unreadable file.
> + return false;
> + }
> + return digest(Buf->get()->getBuffer()) != LS.Digest;
> +}
> +
> } // namespace
>
> BackgroundIndex::BackgroundIndex(
> @@ -156,7 +175,7 @@ BackgroundQueue::Task BackgroundIndex::c
> log("Enqueueing {0} commands for indexing", ChangedFiles.size());
> SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
>
> - auto NeedsReIndexing = loadShards(std::move(ChangedFiles));
> + auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
> // Run indexing for files that need to be updated.
> std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
> std::mt19937(std::random_device{}()));
> @@ -431,169 +450,77 @@ llvm::Error BackgroundIndex::index(tooli
> return llvm::Error::success();
> }
>
> -std::vector<BackgroundIndex::Source>
> -BackgroundIndex::loadShard(const tooling::CompileCommand &Cmd,
> - BackgroundIndexStorage *IndexStorage,
> - llvm::StringSet<> &LoadedShards) {
> - struct ShardInfo {
> - std::string AbsolutePath;
> - std::unique_ptr<IndexFileIn> Shard;
> - FileDigest Digest = {};
> - bool CountReferences = false;
> - bool HadErrors = false;
> - };
> - std::vector<ShardInfo> IntermediateSymbols;
> - // Make sure we don't have duplicate elements in the queue. Keys are absolute
> - // paths.
> - llvm::StringSet<> InQueue;
> - auto FS = FSProvider.getFileSystem();
> - // Dependencies of this TU, paired with the information about whether they
> - // need to be re-indexed or not.
> - std::vector<Source> Dependencies;
> - std::queue<Source> ToVisit;
> - std::string AbsolutePath = getAbsolutePath(Cmd).str();
> - // Up until we load the shard related to a dependency it needs to be
> - // re-indexed.
> - ToVisit.emplace(AbsolutePath, true);
> - InQueue.insert(AbsolutePath);
> - // Goes over each dependency.
> - while (!ToVisit.empty()) {
> - Dependencies.push_back(std::move(ToVisit.front()));
> - // Dependencies is not modified during the rest of the loop, so it is safe
> - // to keep the reference.
> - auto &CurDependency = Dependencies.back();
> - ToVisit.pop();
> - // If we have already seen this shard before(either loaded or failed) don't
> - // re-try again. Since the information in the shard won't change from one TU
> - // to another.
> - if (!LoadedShards.try_emplace(CurDependency.Path).second) {
> - // If the dependency needs to be re-indexed, first occurence would already
> - // have detected that, so we don't need to issue it again.
> - CurDependency.NeedsReIndexing = false;
> - continue;
> - }
> -
> - auto Shard = IndexStorage->loadShard(CurDependency.Path);
> - if (!Shard || !Shard->Sources) {
> - // File will be returned as requiring re-indexing to caller.
> - vlog("Failed to load shard: {0}", CurDependency.Path);
> - continue;
> - }
> - // These are the edges in the include graph for current dependency.
> - for (const auto &I : *Shard->Sources) {
> - auto U = URI::parse(I.getKey());
> - if (!U)
> - continue;
> - auto AbsolutePath = URI::resolve(*U, CurDependency.Path);
> - if (!AbsolutePath)
> - continue;
> - // Add file as dependency if haven't seen before.
> - if (InQueue.try_emplace(*AbsolutePath).second)
> - ToVisit.emplace(*AbsolutePath, true);
> - // The node contains symbol information only for current file, the rest is
> - // just edges.
> - if (*AbsolutePath != CurDependency.Path)
> - continue;
> +// Restores shards for \p MainFiles from index storage. Then checks staleness of
> +// those shards and returns a list of TUs that needs to be indexed to update
> +// staleness.
> +std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
> +BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
> + std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
> + NeedsReIndexing;
>
> - // We found source file info for current dependency.
> - assert(I.getValue().Digest != FileDigest{{0}} && "Digest is empty?");
> - ShardInfo SI;
> - SI.AbsolutePath = CurDependency.Path;
> - SI.Shard = std::move(Shard);
> - SI.Digest = I.getValue().Digest;
> - SI.CountReferences =
> - I.getValue().Flags & IncludeGraphNode::SourceFlag::IsTU;
> - SI.HadErrors =
> - I.getValue().Flags & IncludeGraphNode::SourceFlag::HadErrors;
> - IntermediateSymbols.push_back(std::move(SI));
> - // Check if the source needs re-indexing.
> - // Get the digest, skip it if file doesn't exist.
> - auto Buf = FS->getBufferForFile(CurDependency.Path);
> - if (!Buf) {
> - elog("Couldn't get buffer for file: {0}: {1}", CurDependency.Path,
> - Buf.getError().message());
> - continue;
> - }
> - // If digests match then dependency doesn't need re-indexing.
> - // FIXME: Also check for dependencies(sources) of this shard and compile
> - // commands for cache invalidation.
> - CurDependency.NeedsReIndexing =
> - digest(Buf->get()->getBuffer()) != I.getValue().Digest;
> - }
> - }
> - // Load shard information into background-index.
> + Rebuilder.startLoading();
> + // Load shards for all of the mainfiles.
> + const std::vector<LoadedShard> Result =
> + loadIndexShards(MainFiles, IndexStorageFactory, CDB);
> + size_t LoadedShards = 0;
> {
> + // Update in-memory state.
> std::lock_guard<std::mutex> Lock(ShardVersionsMu);
> - // This can override a newer version that is added in another thread,
> - // if this thread sees the older version but finishes later. This
> - // should be rare in practice.
> - for (const ShardInfo &SI : IntermediateSymbols) {
> + for (auto &LS : Result) {
> + if (!LS.Shard)
> + continue;
> auto SS =
> - SI.Shard->Symbols
> - ? llvm::make_unique<SymbolSlab>(std::move(*SI.Shard->Symbols))
> + LS.Shard->Symbols
> + ? llvm::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))
> : nullptr;
> - auto RS = SI.Shard->Refs
> - ? llvm::make_unique<RefSlab>(std::move(*SI.Shard->Refs))
> + auto RS = LS.Shard->Refs
> + ? llvm::make_unique<RefSlab>(std::move(*LS.Shard->Refs))
> : nullptr;
> auto RelS =
> - SI.Shard->Relations
> - ? llvm::make_unique<RelationSlab>(std::move(*SI.Shard->Relations))
> + LS.Shard->Relations
> + ? llvm::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))
> : nullptr;
> - ShardVersion &SV = ShardVersions[SI.AbsolutePath];
> - SV.Digest = SI.Digest;
> - SV.HadErrors = SI.HadErrors;
> + ShardVersion &SV = ShardVersions[LS.AbsolutePath];
> + SV.Digest = LS.Digest;
> + SV.HadErrors = LS.HadErrors;
> + ++LoadedShards;
>
> - IndexedSymbols.update(SI.AbsolutePath, std::move(SS), std::move(RS),
> - std::move(RelS), SI.CountReferences);
> + IndexedSymbols.update(LS.AbsolutePath, std::move(SS), std::move(RS),
> + std::move(RelS), LS.CountReferences);
> }
> }
> - if (!IntermediateSymbols.empty())
> - Rebuilder.loadedTU();
> + Rebuilder.loadedShard(LoadedShards);
> + Rebuilder.doneLoading();
>
> - return Dependencies;
> -}
> + auto FS = FSProvider.getFileSystem();
> + llvm::DenseSet<PathRef> TUsToIndex;
> + // We'll accept data from stale shards, but ensure the files get reindexed
> + // soon.
> + for (auto &LS : Result) {
> + if (!shardIsStale(LS, FS.get()))
> + continue;
> + PathRef TUForFile = LS.DependentTU;
> + assert(!TUForFile.empty() && "File without a TU!");
>
> -// Goes over each changed file and loads them from index. Returns the list of
> -// TUs that had out-of-date/no shards.
> -std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
> -BackgroundIndex::loadShards(std::vector<std::string> ChangedFiles) {
> - std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
> - NeedsReIndexing;
> - // Keeps track of the files that will be reindexed, to make sure we won't
> - // re-index same dependencies more than once. Keys are AbsolutePaths.
> - llvm::StringSet<> FilesToIndex;
> - // Keeps track of the loaded shards to make sure we don't perform redundant
> - // disk IO. Keys are absolute paths.
> - llvm::StringSet<> LoadedShards;
> - Rebuilder.startLoading();
> - for (const auto &File : ChangedFiles) {
> - auto Cmd = CDB.getCompileCommand(File);
> + // FIXME: Currently, we simply schedule indexing on a TU whenever any of
> + // its dependencies needs re-indexing. We might do it smarter by figuring
> + // out a minimal set of TUs that will cover all the stale dependencies.
> + // FIXME: Try looking at other TUs if no compile commands are available
> + // for this TU, i.e TU was deleted after we performed indexing.
> + TUsToIndex.insert(TUForFile);
> + }
> +
> + for (PathRef TU : TUsToIndex) {
> + auto Cmd = CDB.getCompileCommand(TU);
> if (!Cmd)
> continue;
> -
> std::string ProjectRoot;
> - if (auto PI = CDB.getProjectInfo(File))
> + if (auto PI = CDB.getProjectInfo(TU))
> ProjectRoot = std::move(PI->SourceRoot);
> -
> - BackgroundIndexStorage *IndexStorage = IndexStorageFactory(ProjectRoot);
> - auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);
> - for (const auto &Dependency : Dependencies) {
> - if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))
> - continue;
> - // FIXME: Currently, we simply schedule indexing on a TU whenever any of
> - // its dependencies needs re-indexing. We might do it smarter by figuring
> - // out a minimal set of TUs that will cover all the stale dependencies.
> - vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
> - Cmd->Filename, Dependency.Path);
> - NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});
> - // Mark all of this TU's dependencies as to-be-indexed so that we won't
> - // try to re-index those.
> - for (const auto &Dependency : Dependencies)
> - FilesToIndex.insert(Dependency.Path);
> - break;
> - }
> + BackgroundIndexStorage *Storage = IndexStorageFactory(ProjectRoot);
> + NeedsReIndexing.emplace_back(std::move(*Cmd), Storage);
> }
> - Rebuilder.doneLoading();
> +
> return NeedsReIndexing;
> }
>
>
> Modified: clang-tools-extra/trunk/clangd/index/Background.h
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.h?rev=366458&r1=366457&r2=366458&view=diff
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/Background.h (original)
> +++ clang-tools-extra/trunk/clangd/index/Background.h Thu Jul 18 09:25:36 2019
> @@ -12,6 +12,7 @@
> #include "Context.h"
> #include "FSProvider.h"
> #include "GlobalCompilationDatabase.h"
> +#include "Path.h"
> #include "SourceCode.h"
> #include "Threading.h"
> #include "index/BackgroundRebuild.h"
> @@ -173,20 +174,9 @@ private:
> std::mutex ShardVersionsMu;
>
> BackgroundIndexStorage::Factory IndexStorageFactory;
> - struct Source {
> - std::string Path;
> - bool NeedsReIndexing;
> - Source(llvm::StringRef Path, bool NeedsReIndexing)
> - : Path(Path), NeedsReIndexing(NeedsReIndexing) {}
> - };
> - // Loads the shards for a single TU and all of its dependencies. Returns the
> - // list of sources and whether they need to be re-indexed.
> - std::vector<Source> loadShard(const tooling::CompileCommand &Cmd,
> - BackgroundIndexStorage *IndexStorage,
> - llvm::StringSet<> &LoadedShards);
> - // Tries to load shards for the ChangedFiles.
> + // Tries to load shards for the MainFiles and their dependencies.
> std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
> - loadShards(std::vector<std::string> ChangedFiles);
> + loadProject(std::vector<std::string> MainFiles);
>
> BackgroundQueue::Task
> changedFilesTask(const std::vector<std::string> &ChangedFiles);
>
> Added: clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp?rev=366458&view=auto
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp (added)
> +++ clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp Thu Jul 18 09:25:36 2019
> @@ -0,0 +1,153 @@
> +//===-- BackgroundIndexLoader.cpp - ---------------------------------------===//
> +//
> +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
> +// See https://llvm.org/LICENSE.txt for license information.
> +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
> +//
> +//===----------------------------------------------------------------------===//
> +
> +#include "index/BackgroundIndexLoader.h"
> +#include "GlobalCompilationDatabase.h"
> +#include "Logger.h"
> +#include "Path.h"
> +#include "index/Background.h"
> +#include "llvm/ADT/DenseMap.h"
> +#include "llvm/ADT/DenseSet.h"
> +#include "llvm/ADT/SmallString.h"
> +#include "llvm/ADT/StringMap.h"
> +#include "llvm/Support/Path.h"
> +#include <string>
> +#include <utility>
> +#include <vector>
> +
> +namespace clang {
> +namespace clangd {
> +namespace {
> +
> +llvm::Optional<Path> uriToAbsolutePath(llvm::StringRef URI, PathRef HintPath) {
> + auto U = URI::parse(URI);
> + if (!U)
> + return llvm::None;
> + auto AbsolutePath = URI::resolve(*U, HintPath);
> + if (!AbsolutePath)
> + return llvm::None;
> + return *AbsolutePath;
> +}
> +
> +/// A helper class to cache BackgroundIndexStorage operations and keep the
> +/// inverse dependency mapping.
> +class BackgroundIndexLoader {
> +public:
> + /// Load the shards for \p MainFile and all of its dependencies.
> + void load(PathRef MainFile, BackgroundIndexStorage *Storage);
> +
> + /// Consumes the loader and returns all shards.
> + std::vector<LoadedShard> takeResult() &&;
> +
> +private:
> + /// Returns the Shard for \p StartSourceFile from cache or loads it from \p
> + /// Storage. Also returns paths for dependencies of \p StartSourceFile if it
> + /// wasn't cached yet.
> + std::pair<const LoadedShard &, std::vector<Path>>
> + loadShard(PathRef StartSourceFile, BackgroundIndexStorage *Storage);
> +
> + /// Cache for Storage lookups.
> + llvm::StringMap<LoadedShard> LoadedShards;
> +
> + /// References are into the AbsolutePaths in LoadedShards.
> + llvm::DenseMap<PathRef, PathRef> FileToTU;
> +};
> +
> +std::pair<const LoadedShard &, std::vector<Path>>
> +BackgroundIndexLoader::loadShard(PathRef StartSourceFile,
> + BackgroundIndexStorage *Storage) {
> + auto It = LoadedShards.try_emplace(StartSourceFile);
> + LoadedShard &LS = It.first->getValue();
> + std::vector<Path> Edges = {};
> + // Return the cached shard.
> + if (!It.second)
> + return {LS, Edges};
> +
> + LS.AbsolutePath = StartSourceFile.str();
> + auto Shard = Storage->loadShard(StartSourceFile);
> + if (!Shard || !Shard->Sources) {
> + vlog("Failed to load shard: {0}", StartSourceFile);
> + return {LS, Edges};
> + }
> +
> + LS.Shard = std::move(Shard);
> + for (const auto &It : *LS.Shard->Sources) {
> + auto AbsPath = uriToAbsolutePath(It.getKey(), StartSourceFile);
> + if (!AbsPath)
> + continue;
> + // A shard contains only edges for non main-file sources.
> + if (*AbsPath != StartSourceFile) {
> + Edges.push_back(*AbsPath);
> + continue;
> + }
> +
> + // Fill in shard metadata.
> + const IncludeGraphNode &IGN = It.getValue();
> + LS.Digest = IGN.Digest;
> + LS.CountReferences = IGN.Flags & IncludeGraphNode::SourceFlag::IsTU;
> + LS.HadErrors = IGN.Flags & IncludeGraphNode::SourceFlag::HadErrors;
> + }
> + assert(LS.Digest != FileDigest{{0}} && "Digest is empty?");
> + return {LS, Edges};
> +}
> +
> +void BackgroundIndexLoader::load(PathRef MainFile,
> + BackgroundIndexStorage *Storage) {
> + llvm::StringSet<> InQueue;
> + // Following containers points to strings inside InQueue.
> + std::queue<PathRef> ToVisit;
> + InQueue.insert(MainFile);
> + ToVisit.push(MainFile);
> +
> + while (!ToVisit.empty()) {
> + PathRef SourceFile = ToVisit.front();
> + ToVisit.pop();
> +
> + auto ShardAndEdges = loadShard(SourceFile, Storage);
> + FileToTU[ShardAndEdges.first.AbsolutePath] = MainFile;
> + for (PathRef Edge : ShardAndEdges.second) {
> + auto It = InQueue.insert(Edge);
> + if (It.second)
> + ToVisit.push(It.first->getKey());
> + }
> + }
> +}
> +
> +std::vector<LoadedShard> BackgroundIndexLoader::takeResult() && {
> + std::vector<LoadedShard> Result;
> + Result.reserve(LoadedShards.size());
> + for (auto &It : LoadedShards) {
> + Result.push_back(std::move(It.getValue()));
> + LoadedShard &LS = Result.back();
> + auto TUsIt = FileToTU.find(LS.AbsolutePath);
> + assert(TUsIt != FileToTU.end() && "No TU registered for the shard");
> + Result.back().DependentTU = TUsIt->second;
> + }
> + return Result;
> +}
> +} // namespace
> +
> +std::vector<LoadedShard>
> +loadIndexShards(llvm::ArrayRef<Path> MainFiles,
> + BackgroundIndexStorage::Factory &IndexStorageFactory,
> + const GlobalCompilationDatabase &CDB) {
> + BackgroundIndexLoader Loader;
> + for (llvm::StringRef MainFile : MainFiles) {
> + assert(llvm::sys::path::is_absolute(MainFile));
> +
> + std::string ProjectRoot;
> + if (auto PI = CDB.getProjectInfo(MainFile))
> + ProjectRoot = std::move(PI->SourceRoot);
> + BackgroundIndexStorage *Storage = IndexStorageFactory(ProjectRoot);
> + Loader.load(MainFile, Storage);
> + }
> + return std::move(Loader).takeResult();
> +}
> +
> +} // namespace clangd
> +} // namespace clang
>
> Added: clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h?rev=366458&view=auto
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h (added)
> +++ clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h Thu Jul 18 09:25:36 2019
> @@ -0,0 +1,54 @@
> +//===--- BackgroundIndexLoader.h - Load shards from index storage-*- C++-*-===//
> +//
> +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
> +// See https://llvm.org/LICENSE.txt for license information.
> +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
> +//
> +//===----------------------------------------------------------------------===//
> +
> +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_INDEX_LOADER_H
> +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_INDEX_LOADER_H
> +
> +#include "Path.h"
> +#include "index/Background.h"
> +#include "llvm/ADT/ArrayRef.h"
> +#include "llvm/ADT/DenseMap.h"
> +#include "llvm/ADT/Optional.h"
> +#include "llvm/ADT/StringMap.h"
> +#include "llvm/ADT/StringRef.h"
> +#include "llvm/Support/VirtualFileSystem.h"
> +#include <memory>
> +#include <vector>
> +
> +namespace clang {
> +namespace clangd {
> +
> +/// Represents a shard loaded from storage, stores contents in \p Shard and
> +/// metadata about the source file that generated this shard.
> +struct LoadedShard {
> + /// Path of the source file that produced this shard.
> + Path AbsolutePath;
> + /// Digest of the source file contents that produced this shard.
> + FileDigest Digest = {};
> + /// Whether the RefSlab in Shard should be used for updating symbol reference
> + /// counts when building an index.
> + bool CountReferences = false;
> + /// Whether the indexing action producing that shard had errors.
> + bool HadErrors = false;
> + /// Path to a TU that is depending on this shard.
> + Path DependentTU;
> + /// Will be nullptr when index storage couldn't provide a valid shard for
> + /// AbsolutePath.
> + std::unique_ptr<IndexFileIn> Shard;
> +};
> +
> +/// Loads all shards for the TU \p MainFile from \p Storage.
> +std::vector<LoadedShard>
> +loadIndexShards(llvm::ArrayRef<Path> MainFiles,
> + BackgroundIndexStorage::Factory &IndexStorageFactory,
> + const GlobalCompilationDatabase &CDB);
> +
> +} // namespace clangd
> +} // namespace clang
> +
> +#endif
>
> Modified: clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp?rev=366458&r1=366457&r2=366458&view=diff
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp (original)
> +++ clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp Thu Jul 18 09:25:36 2019
> @@ -78,13 +78,13 @@ void BackgroundIndexRebuilder::idle() {
> void BackgroundIndexRebuilder::startLoading() {
> std::lock_guard<std::mutex> Lock(Mu);
> if (!Loading)
> - LoadedTUs = 0;
> + LoadedShards = 0;
> ++Loading;
> }
> -void BackgroundIndexRebuilder::loadedTU() {
> +void BackgroundIndexRebuilder::loadedShard(size_t ShardCount) {
> std::lock_guard<std::mutex> Lock(Mu);
> assert(Loading);
> - ++LoadedTUs;
> + LoadedShards += ShardCount;
> }
> void BackgroundIndexRebuilder::doneLoading() {
> maybeRebuild("after loading index from disk", [this] {
> @@ -93,7 +93,7 @@ void BackgroundIndexRebuilder::doneLoadi
> if (Loading) // was loading multiple batches concurrently
> return false; // rebuild once the last batch is done.
> // Rebuild if we loaded any shards, or if we stopped an indexedTU rebuild.
> - return LoadedTUs > 0 || enoughTUsToRebuild();
> + return LoadedShards > 0 || enoughTUsToRebuild();
> });
> }
>
>
> Modified: clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h?rev=366458&r1=366457&r2=366458&view=diff
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h (original)
> +++ clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h Thu Jul 18 09:25:36 2019
> @@ -17,6 +17,7 @@
> #include "index/FileIndex.h"
> #include "index/Index.h"
> #include "llvm/Support/Threading.h"
> +#include <cstddef>
>
> namespace clang {
> namespace clangd {
> @@ -61,7 +62,7 @@ public:
> // sessions may happen concurrently.
> void startLoading();
> // Called to indicate some shards were actually loaded from disk.
> - void loadedTU();
> + void loadedShard(size_t ShardCount);
> // Called to indicate we're finished loading shards from disk.
> // May rebuild (if any were loaded).
> void doneLoading();
> @@ -89,7 +90,7 @@ private:
> unsigned IndexedTUsAtLastRebuild = 0;
> // Are we loading shards? May be multiple concurrent sessions.
> unsigned Loading = 0;
> - unsigned LoadedTUs; // In the current loading session.
> + unsigned LoadedShards; // In the current loading session.
>
> SwapIndex *Target;
> FileSymbols *Source;
>
> Modified: clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp
> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp?rev=366458&r1=366457&r2=366458&view=diff
> ==============================================================================
> --- clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp (original)
> +++ clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp Thu Jul 18 09:25:36 2019
> @@ -211,7 +211,7 @@ TEST_F(BackgroundIndexTest, ShardStorage
> OverlayCDB CDB(/*Base=*/nullptr);
> BackgroundIndex Idx(Context::empty(), FS, CDB,
> [&](llvm::StringRef) { return &MSS; });
> - CDB.setCompileCommand(testPath("root"), Cmd);
> + CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
> ASSERT_TRUE(Idx.blockUntilIdleForTest());
> }
> EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
> @@ -335,7 +335,7 @@ TEST_F(BackgroundIndexTest, ShardStorage
> OverlayCDB CDB(/*Base=*/nullptr);
> BackgroundIndex Idx(Context::empty(), FS, CDB,
> [&](llvm::StringRef) { return &MSS; });
> - CDB.setCompileCommand(testPath("root"), Cmd);
> + CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
> ASSERT_TRUE(Idx.blockUntilIdleForTest());
> }
> EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
> @@ -353,7 +353,7 @@ TEST_F(BackgroundIndexTest, ShardStorage
> OverlayCDB CDB(/*Base=*/nullptr);
> BackgroundIndex Idx(Context::empty(), FS, CDB,
> [&](llvm::StringRef) { return &MSS; });
> - CDB.setCompileCommand(testPath("root"), Cmd);
> + CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
> ASSERT_TRUE(Idx.blockUntilIdleForTest());
> }
> EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
> @@ -621,8 +621,8 @@ TEST_F(BackgroundIndexRebuilderTest, Ind
>
> TEST_F(BackgroundIndexRebuilderTest, LoadingShards) {
> Rebuilder.startLoading();
> - Rebuilder.loadedTU();
> - Rebuilder.loadedTU();
> + Rebuilder.loadedShard(10);
> + Rebuilder.loadedShard(20);
> EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
>
> // No rebuild for no shards.
> @@ -631,11 +631,11 @@ TEST_F(BackgroundIndexRebuilderTest, Loa
>
> // Loads can overlap.
> Rebuilder.startLoading();
> - Rebuilder.loadedTU();
> + Rebuilder.loadedShard(1);
> Rebuilder.startLoading();
> - Rebuilder.loadedTU();
> + Rebuilder.loadedShard(1);
> EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
> - Rebuilder.loadedTU();
> + Rebuilder.loadedShard(1);
> EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
>
> // No rebuilding for indexed files while loading.
>
>
> _______________________________________________
> cfe-commits mailing list
> cfe-commits at lists.llvm.org
> https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20190718/0a7a5947/attachment-0001.html>
More information about the cfe-commits
mailing list