[clang-tools-extra] r366458 - [clangd] Refactor background-index shard loading

Azhar Mohammed via cfe-commits cfe-commits at lists.llvm.org
Fri Jul 19 02:29:34 PDT 2019


Reverted in r366551. 


    Revert r366458, r366467 and r366468

    r366458 is causing test failures. r366467 and r366468 had to be reverted as
    they were casuing conflict while reverting r366458.

    r366468 [clangd] Remove dead code from BackgroundIndex
    r366467 [clangd] BackgroundIndex stores shards to the closest project
    r366458 [clangd] Refactor background-index shard loading

> On Jul 18, 2019, at 6:21 PM, Azhar Mohammed <azhar at apple.com> wrote:
> 
> Hi Kadir
> 
> This change is causing test failures, can you please look into it. Refer to http://green.lab.llvm.org/green/job/clang-stage1-configure-RA/58104/testReport/ <http://green.lab.llvm.org/green/job/clang-stage1-configure-RA/58104/testReport/>. 
> 
> Assertion failed: (TUsIt != FileToTU.end() && "No TU registered for the shard"), function takeResult, file /Users/buildslave/jenkins/workspace/clang-stage1-configure-RA/llvm/tools/clang/tools/extra/clangd/index/BackgroundIndexLoader.cpp, line 131.
> 
> 
> Failing Tests (10):
>     Clangd :: did-change-configuration-params.test
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.CmdLineHash
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.DirectIncludesTest
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.IndexTwoFiles
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.NoCrashOnErrorFile
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.NoDotsInAbsPath
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.ShardStorageEmptyFile
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.ShardStorageLoad
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.ShardStorageTest
>     Clangd Unit Tests :: ./ClangdTests/BackgroundIndexTest.UncompilableFiles
> 
> 
>> On Jul 18, 2019, at 9:25 AM, Kadir Cetinkaya via cfe-commits <cfe-commits at lists.llvm.org <mailto:cfe-commits at lists.llvm.org>> wrote:
>> 
>> Author: kadircet
>> Date: Thu Jul 18 09:25:36 2019
>> New Revision: 366458
>> 
>> URL: http://llvm.org/viewvc/llvm-project?rev=366458&view=rev <http://llvm.org/viewvc/llvm-project?rev=366458&view=rev>
>> Log:
>> [clangd] Refactor background-index shard loading
>> 
>> Reviewers: sammccall
>> 
>> Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, cfe-commits
>> 
>> Tags: #clang
>> 
>> Differential Revision: https://reviews.llvm.org/D64712 <https://reviews.llvm.org/D64712>
>> 
>> Added:
>>    clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp
>>    clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h
>> Modified:
>>    clang-tools-extra/trunk/clangd/CMakeLists.txt
>>    clang-tools-extra/trunk/clangd/index/Background.cpp
>>    clang-tools-extra/trunk/clangd/index/Background.h
>>    clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp
>>    clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h
>>    clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp
>> 
>> Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
>> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=366458&r1=366457&r2=366458&view=diff <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=366458&r1=366457&r2=366458&view=diff>
>> ==============================================================================
>> --- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
>> +++ clang-tools-extra/trunk/clangd/CMakeLists.txt Thu Jul 18 09:25:36 2019
>> @@ -73,6 +73,7 @@ add_clang_library(clangDaemon
>>   XRefs.cpp
>> 
>>   index/Background.cpp
>> +  index/BackgroundIndexLoader.cpp
>>   index/BackgroundIndexStorage.cpp
>>   index/BackgroundQueue.cpp
>>   index/BackgroundRebuild.cpp
>> 
>> Modified: clang-tools-extra/trunk/clangd/index/Background.cpp
>> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.cpp?rev=366458&r1=366457&r2=366458&view=diff <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.cpp?rev=366458&r1=366457&r2=366458&view=diff>
>> ==============================================================================
>> --- clang-tools-extra/trunk/clangd/index/Background.cpp (original)
>> +++ clang-tools-extra/trunk/clangd/index/Background.cpp Thu Jul 18 09:25:36 2019
>> @@ -10,6 +10,7 @@
>> #include "ClangdUnit.h"
>> #include "Compiler.h"
>> #include "Context.h"
>> +#include "FSProvider.h"
>> #include "Headers.h"
>> #include "Logger.h"
>> #include "Path.h"
>> @@ -18,6 +19,7 @@
>> #include "Threading.h"
>> #include "Trace.h"
>> #include "URI.h"
>> +#include "index/BackgroundIndexLoader.h"
>> #include "index/FileIndex.h"
>> #include "index/IndexAction.h"
>> #include "index/MemIndex.h"
>> @@ -28,6 +30,8 @@
>> #include "clang/Basic/SourceLocation.h"
>> #include "clang/Basic/SourceManager.h"
>> #include "clang/Driver/Types.h"
>> +#include "llvm/ADT/ArrayRef.h"
>> +#include "llvm/ADT/DenseSet.h"
>> #include "llvm/ADT/Hashing.h"
>> #include "llvm/ADT/STLExtras.h"
>> #include "llvm/ADT/ScopeExit.h"
>> @@ -42,6 +46,7 @@
>> #include <atomic>
>> #include <chrono>
>> #include <condition_variable>
>> +#include <cstddef>
>> #include <memory>
>> #include <mutex>
>> #include <numeric>
>> @@ -49,6 +54,8 @@
>> #include <random>
>> #include <string>
>> #include <thread>
>> +#include <utility>
>> +#include <vector>
>> 
>> namespace clang {
>> namespace clangd {
>> @@ -119,6 +126,18 @@ llvm::SmallString<128> getAbsolutePath(c
>>   }
>>   return AbsolutePath;
>> }
>> +
>> +bool shardIsStale(const LoadedShard &LS, llvm::vfs::FileSystem *FS) {
>> +  auto Buf = FS->getBufferForFile(LS.AbsolutePath);
>> +  if (!Buf) {
>> +    elog("Background-index: Couldn't read {0} to validate stored index: {1}",
>> +         LS.AbsolutePath, Buf.getError().message());
>> +    // There is no point in indexing an unreadable file.
>> +    return false;
>> +  }
>> +  return digest(Buf->get()->getBuffer()) != LS.Digest;
>> +}
>> +
>> } // namespace
>> 
>> BackgroundIndex::BackgroundIndex(
>> @@ -156,7 +175,7 @@ BackgroundQueue::Task BackgroundIndex::c
>>     log("Enqueueing {0} commands for indexing", ChangedFiles.size());
>>     SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
>> 
>> -    auto NeedsReIndexing = loadShards(std::move(ChangedFiles));
>> +    auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
>>     // Run indexing for files that need to be updated.
>>     std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
>>                  std::mt19937(std::random_device{}()));
>> @@ -431,169 +450,77 @@ llvm::Error BackgroundIndex::index(tooli
>>   return llvm::Error::success();
>> }
>> 
>> -std::vector<BackgroundIndex::Source>
>> -BackgroundIndex::loadShard(const tooling::CompileCommand &Cmd,
>> -                           BackgroundIndexStorage *IndexStorage,
>> -                           llvm::StringSet<> &LoadedShards) {
>> -  struct ShardInfo {
>> -    std::string AbsolutePath;
>> -    std::unique_ptr<IndexFileIn> Shard;
>> -    FileDigest Digest = {};
>> -    bool CountReferences = false;
>> -    bool HadErrors = false;
>> -  };
>> -  std::vector<ShardInfo> IntermediateSymbols;
>> -  // Make sure we don't have duplicate elements in the queue. Keys are absolute
>> -  // paths.
>> -  llvm::StringSet<> InQueue;
>> -  auto FS = FSProvider.getFileSystem();
>> -  // Dependencies of this TU, paired with the information about whether they
>> -  // need to be re-indexed or not.
>> -  std::vector<Source> Dependencies;
>> -  std::queue<Source> ToVisit;
>> -  std::string AbsolutePath = getAbsolutePath(Cmd).str();
>> -  // Up until we load the shard related to a dependency it needs to be
>> -  // re-indexed.
>> -  ToVisit.emplace(AbsolutePath, true);
>> -  InQueue.insert(AbsolutePath);
>> -  // Goes over each dependency.
>> -  while (!ToVisit.empty()) {
>> -    Dependencies.push_back(std::move(ToVisit.front()));
>> -    // Dependencies is not modified during the rest of the loop, so it is safe
>> -    // to keep the reference.
>> -    auto &CurDependency = Dependencies.back();
>> -    ToVisit.pop();
>> -    // If we have already seen this shard before(either loaded or failed) don't
>> -    // re-try again. Since the information in the shard won't change from one TU
>> -    // to another.
>> -    if (!LoadedShards.try_emplace(CurDependency.Path).second) {
>> -      // If the dependency needs to be re-indexed, first occurence would already
>> -      // have detected that, so we don't need to issue it again.
>> -      CurDependency.NeedsReIndexing = false;
>> -      continue;
>> -    }
>> -
>> -    auto Shard = IndexStorage->loadShard(CurDependency.Path);
>> -    if (!Shard || !Shard->Sources) {
>> -      // File will be returned as requiring re-indexing to caller.
>> -      vlog("Failed to load shard: {0}", CurDependency.Path);
>> -      continue;
>> -    }
>> -    // These are the edges in the include graph for current dependency.
>> -    for (const auto &I : *Shard->Sources) {
>> -      auto U = URI::parse(I.getKey());
>> -      if (!U)
>> -        continue;
>> -      auto AbsolutePath = URI::resolve(*U, CurDependency.Path);
>> -      if (!AbsolutePath)
>> -        continue;
>> -      // Add file as dependency if haven't seen before.
>> -      if (InQueue.try_emplace(*AbsolutePath).second)
>> -        ToVisit.emplace(*AbsolutePath, true);
>> -      // The node contains symbol information only for current file, the rest is
>> -      // just edges.
>> -      if (*AbsolutePath != CurDependency.Path)
>> -        continue;
>> +// Restores shards for \p MainFiles from index storage. Then checks staleness of
>> +// those shards and returns a list of TUs that needs to be indexed to update
>> +// staleness.
>> +std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
>> +BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
>> +  std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
>> +      NeedsReIndexing;
>> 
>> -      // We found source file info for current dependency.
>> -      assert(I.getValue().Digest != FileDigest{{0}} && "Digest is empty?");
>> -      ShardInfo SI;
>> -      SI.AbsolutePath = CurDependency.Path;
>> -      SI.Shard = std::move(Shard);
>> -      SI.Digest = I.getValue().Digest;
>> -      SI.CountReferences =
>> -          I.getValue().Flags & IncludeGraphNode::SourceFlag::IsTU;
>> -      SI.HadErrors =
>> -          I.getValue().Flags & IncludeGraphNode::SourceFlag::HadErrors;
>> -      IntermediateSymbols.push_back(std::move(SI));
>> -      // Check if the source needs re-indexing.
>> -      // Get the digest, skip it if file doesn't exist.
>> -      auto Buf = FS->getBufferForFile(CurDependency.Path);
>> -      if (!Buf) {
>> -        elog("Couldn't get buffer for file: {0}: {1}", CurDependency.Path,
>> -             Buf.getError().message());
>> -        continue;
>> -      }
>> -      // If digests match then dependency doesn't need re-indexing.
>> -      // FIXME: Also check for dependencies(sources) of this shard and compile
>> -      // commands for cache invalidation.
>> -      CurDependency.NeedsReIndexing =
>> -          digest(Buf->get()->getBuffer()) != I.getValue().Digest;
>> -    }
>> -  }
>> -  // Load shard information into background-index.
>> +  Rebuilder.startLoading();
>> +  // Load shards for all of the mainfiles.
>> +  const std::vector<LoadedShard> Result =
>> +      loadIndexShards(MainFiles, IndexStorageFactory, CDB);
>> +  size_t LoadedShards = 0;
>>   {
>> +    // Update in-memory state.
>>     std::lock_guard<std::mutex> Lock(ShardVersionsMu);
>> -    // This can override a newer version that is added in another thread,
>> -    // if this thread sees the older version but finishes later. This
>> -    // should be rare in practice.
>> -    for (const ShardInfo &SI : IntermediateSymbols) {
>> +    for (auto &LS : Result) {
>> +      if (!LS.Shard)
>> +        continue;
>>       auto SS =
>> -          SI.Shard->Symbols
>> -              ? llvm::make_unique<SymbolSlab>(std::move(*SI.Shard->Symbols))
>> +          LS.Shard->Symbols
>> +              ? llvm::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))
>>               : nullptr;
>> -      auto RS = SI.Shard->Refs
>> -                    ? llvm::make_unique<RefSlab>(std::move(*SI.Shard->Refs))
>> +      auto RS = LS.Shard->Refs
>> +                    ? llvm::make_unique<RefSlab>(std::move(*LS.Shard->Refs))
>>                     : nullptr;
>>       auto RelS =
>> -          SI.Shard->Relations
>> -              ? llvm::make_unique<RelationSlab>(std::move(*SI.Shard->Relations))
>> +          LS.Shard->Relations
>> +              ? llvm::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))
>>               : nullptr;
>> -      ShardVersion &SV = ShardVersions[SI.AbsolutePath];
>> -      SV.Digest = SI.Digest;
>> -      SV.HadErrors = SI.HadErrors;
>> +      ShardVersion &SV = ShardVersions[LS.AbsolutePath];
>> +      SV.Digest = LS.Digest;
>> +      SV.HadErrors = LS.HadErrors;
>> +      ++LoadedShards;
>> 
>> -      IndexedSymbols.update(SI.AbsolutePath, std::move(SS), std::move(RS),
>> -                            std::move(RelS), SI.CountReferences);
>> +      IndexedSymbols.update(LS.AbsolutePath, std::move(SS), std::move(RS),
>> +                            std::move(RelS), LS.CountReferences);
>>     }
>>   }
>> -  if (!IntermediateSymbols.empty())
>> -    Rebuilder.loadedTU();
>> +  Rebuilder.loadedShard(LoadedShards);
>> +  Rebuilder.doneLoading();
>> 
>> -  return Dependencies;
>> -}
>> +  auto FS = FSProvider.getFileSystem();
>> +  llvm::DenseSet<PathRef> TUsToIndex;
>> +  // We'll accept data from stale shards, but ensure the files get reindexed
>> +  // soon.
>> +  for (auto &LS : Result) {
>> +    if (!shardIsStale(LS, FS.get()))
>> +      continue;
>> +    PathRef TUForFile = LS.DependentTU;
>> +    assert(!TUForFile.empty() && "File without a TU!");
>> 
>> -// Goes over each changed file and loads them from index. Returns the list of
>> -// TUs that had out-of-date/no shards.
>> -std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
>> -BackgroundIndex::loadShards(std::vector<std::string> ChangedFiles) {
>> -  std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
>> -      NeedsReIndexing;
>> -  // Keeps track of the files that will be reindexed, to make sure we won't
>> -  // re-index same dependencies more than once. Keys are AbsolutePaths.
>> -  llvm::StringSet<> FilesToIndex;
>> -  // Keeps track of the loaded shards to make sure we don't perform redundant
>> -  // disk IO. Keys are absolute paths.
>> -  llvm::StringSet<> LoadedShards;
>> -  Rebuilder.startLoading();
>> -  for (const auto &File : ChangedFiles) {
>> -    auto Cmd = CDB.getCompileCommand(File);
>> +    // FIXME: Currently, we simply schedule indexing on a TU whenever any of
>> +    // its dependencies needs re-indexing. We might do it smarter by figuring
>> +    // out a minimal set of TUs that will cover all the stale dependencies.
>> +    // FIXME: Try looking at other TUs if no compile commands are available
>> +    // for this TU, i.e TU was deleted after we performed indexing.
>> +    TUsToIndex.insert(TUForFile);
>> +  }
>> +
>> +  for (PathRef TU : TUsToIndex) {
>> +    auto Cmd = CDB.getCompileCommand(TU);
>>     if (!Cmd)
>>       continue;
>> -
>>     std::string ProjectRoot;
>> -    if (auto PI = CDB.getProjectInfo(File))
>> +    if (auto PI = CDB.getProjectInfo(TU))
>>       ProjectRoot = std::move(PI->SourceRoot);
>> -
>> -    BackgroundIndexStorage *IndexStorage = IndexStorageFactory(ProjectRoot);
>> -    auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);
>> -    for (const auto &Dependency : Dependencies) {
>> -      if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))
>> -        continue;
>> -      // FIXME: Currently, we simply schedule indexing on a TU whenever any of
>> -      // its dependencies needs re-indexing. We might do it smarter by figuring
>> -      // out a minimal set of TUs that will cover all the stale dependencies.
>> -      vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
>> -           Cmd->Filename, Dependency.Path);
>> -      NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});
>> -      // Mark all of this TU's dependencies as to-be-indexed so that we won't
>> -      // try to re-index those.
>> -      for (const auto &Dependency : Dependencies)
>> -        FilesToIndex.insert(Dependency.Path);
>> -      break;
>> -    }
>> +    BackgroundIndexStorage *Storage = IndexStorageFactory(ProjectRoot);
>> +    NeedsReIndexing.emplace_back(std::move(*Cmd), Storage);
>>   }
>> -  Rebuilder.doneLoading();
>> +
>>   return NeedsReIndexing;
>> }
>> 
>> 
>> Modified: clang-tools-extra/trunk/clangd/index/Background.h
>> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.h?rev=366458&r1=366457&r2=366458&view=diff <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.h?rev=366458&r1=366457&r2=366458&view=diff>
>> ==============================================================================
>> --- clang-tools-extra/trunk/clangd/index/Background.h (original)
>> +++ clang-tools-extra/trunk/clangd/index/Background.h Thu Jul 18 09:25:36 2019
>> @@ -12,6 +12,7 @@
>> #include "Context.h"
>> #include "FSProvider.h"
>> #include "GlobalCompilationDatabase.h"
>> +#include "Path.h"
>> #include "SourceCode.h"
>> #include "Threading.h"
>> #include "index/BackgroundRebuild.h"
>> @@ -173,20 +174,9 @@ private:
>>   std::mutex ShardVersionsMu;
>> 
>>   BackgroundIndexStorage::Factory IndexStorageFactory;
>> -  struct Source {
>> -    std::string Path;
>> -    bool NeedsReIndexing;
>> -    Source(llvm::StringRef Path, bool NeedsReIndexing)
>> -        : Path(Path), NeedsReIndexing(NeedsReIndexing) {}
>> -  };
>> -  // Loads the shards for a single TU and all of its dependencies. Returns the
>> -  // list of sources and whether they need to be re-indexed.
>> -  std::vector<Source> loadShard(const tooling::CompileCommand &Cmd,
>> -                                BackgroundIndexStorage *IndexStorage,
>> -                                llvm::StringSet<> &LoadedShards);
>> -  // Tries to load shards for the ChangedFiles.
>> +  // Tries to load shards for the MainFiles and their dependencies.
>>   std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
>> -  loadShards(std::vector<std::string> ChangedFiles);
>> +  loadProject(std::vector<std::string> MainFiles);
>> 
>>   BackgroundQueue::Task
>>   changedFilesTask(const std::vector<std::string> &ChangedFiles);
>> 
>> Added: clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp
>> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp?rev=366458&view=auto <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp?rev=366458&view=auto>
>> ==============================================================================
>> --- clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp (added)
>> +++ clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.cpp Thu Jul 18 09:25:36 2019
>> @@ -0,0 +1,153 @@
>> +//===-- BackgroundIndexLoader.cpp - ---------------------------------------===//
>> +//
>> +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
>> +// See https://llvm.org/LICENSE.txt <https://llvm.org/LICENSE.txt> for license information.
>> +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
>> +//
>> +//===----------------------------------------------------------------------===//
>> +
>> +#include "index/BackgroundIndexLoader.h"
>> +#include "GlobalCompilationDatabase.h"
>> +#include "Logger.h"
>> +#include "Path.h"
>> +#include "index/Background.h"
>> +#include "llvm/ADT/DenseMap.h"
>> +#include "llvm/ADT/DenseSet.h"
>> +#include "llvm/ADT/SmallString.h"
>> +#include "llvm/ADT/StringMap.h"
>> +#include "llvm/Support/Path.h"
>> +#include <string>
>> +#include <utility>
>> +#include <vector>
>> +
>> +namespace clang {
>> +namespace clangd {
>> +namespace {
>> +
>> +llvm::Optional<Path> uriToAbsolutePath(llvm::StringRef URI, PathRef HintPath) {
>> +  auto U = URI::parse(URI);
>> +  if (!U)
>> +    return llvm::None;
>> +  auto AbsolutePath = URI::resolve(*U, HintPath);
>> +  if (!AbsolutePath)
>> +    return llvm::None;
>> +  return *AbsolutePath;
>> +}
>> +
>> +/// A helper class to cache BackgroundIndexStorage operations and keep the
>> +/// inverse dependency mapping.
>> +class BackgroundIndexLoader {
>> +public:
>> +  /// Load the shards for \p MainFile and all of its dependencies.
>> +  void load(PathRef MainFile, BackgroundIndexStorage *Storage);
>> +
>> +  /// Consumes the loader and returns all shards.
>> +  std::vector<LoadedShard> takeResult() &&;
>> +
>> +private:
>> +  /// Returns the Shard for \p StartSourceFile from cache or loads it from \p
>> +  /// Storage. Also returns paths for dependencies of \p StartSourceFile if it
>> +  /// wasn't cached yet.
>> +  std::pair<const LoadedShard &, std::vector<Path>>
>> +  loadShard(PathRef StartSourceFile, BackgroundIndexStorage *Storage);
>> +
>> +  /// Cache for Storage lookups.
>> +  llvm::StringMap<LoadedShard> LoadedShards;
>> +
>> +  /// References are into the AbsolutePaths in LoadedShards.
>> +  llvm::DenseMap<PathRef, PathRef> FileToTU;
>> +};
>> +
>> +std::pair<const LoadedShard &, std::vector<Path>>
>> +BackgroundIndexLoader::loadShard(PathRef StartSourceFile,
>> +                                 BackgroundIndexStorage *Storage) {
>> +  auto It = LoadedShards.try_emplace(StartSourceFile);
>> +  LoadedShard &LS = It.first->getValue();
>> +  std::vector<Path> Edges = {};
>> +  // Return the cached shard.
>> +  if (!It.second)
>> +    return {LS, Edges};
>> +
>> +  LS.AbsolutePath = StartSourceFile.str();
>> +  auto Shard = Storage->loadShard(StartSourceFile);
>> +  if (!Shard || !Shard->Sources) {
>> +    vlog("Failed to load shard: {0}", StartSourceFile);
>> +    return {LS, Edges};
>> +  }
>> +
>> +  LS.Shard = std::move(Shard);
>> +  for (const auto &It : *LS.Shard->Sources) {
>> +    auto AbsPath = uriToAbsolutePath(It.getKey(), StartSourceFile);
>> +    if (!AbsPath)
>> +      continue;
>> +    // A shard contains only edges for non main-file sources.
>> +    if (*AbsPath != StartSourceFile) {
>> +      Edges.push_back(*AbsPath);
>> +      continue;
>> +    }
>> +
>> +    // Fill in shard metadata.
>> +    const IncludeGraphNode &IGN = It.getValue();
>> +    LS.Digest = IGN.Digest;
>> +    LS.CountReferences = IGN.Flags & IncludeGraphNode::SourceFlag::IsTU;
>> +    LS.HadErrors = IGN.Flags & IncludeGraphNode::SourceFlag::HadErrors;
>> +  }
>> +  assert(LS.Digest != FileDigest{{0}} && "Digest is empty?");
>> +  return {LS, Edges};
>> +}
>> +
>> +void BackgroundIndexLoader::load(PathRef MainFile,
>> +                                 BackgroundIndexStorage *Storage) {
>> +  llvm::StringSet<> InQueue;
>> +  // Following containers points to strings inside InQueue.
>> +  std::queue<PathRef> ToVisit;
>> +  InQueue.insert(MainFile);
>> +  ToVisit.push(MainFile);
>> +
>> +  while (!ToVisit.empty()) {
>> +    PathRef SourceFile = ToVisit.front();
>> +    ToVisit.pop();
>> +
>> +    auto ShardAndEdges = loadShard(SourceFile, Storage);
>> +    FileToTU[ShardAndEdges.first.AbsolutePath] = MainFile;
>> +    for (PathRef Edge : ShardAndEdges.second) {
>> +      auto It = InQueue.insert(Edge);
>> +      if (It.second)
>> +        ToVisit.push(It.first->getKey());
>> +    }
>> +  }
>> +}
>> +
>> +std::vector<LoadedShard> BackgroundIndexLoader::takeResult() && {
>> +  std::vector<LoadedShard> Result;
>> +  Result.reserve(LoadedShards.size());
>> +  for (auto &It : LoadedShards) {
>> +    Result.push_back(std::move(It.getValue()));
>> +    LoadedShard &LS = Result.back();
>> +    auto TUsIt = FileToTU.find(LS.AbsolutePath);
>> +    assert(TUsIt != FileToTU.end() && "No TU registered for the shard");
>> +    Result.back().DependentTU = TUsIt->second;
>> +  }
>> +  return Result;
>> +}
>> +} // namespace
>> +
>> +std::vector<LoadedShard>
>> +loadIndexShards(llvm::ArrayRef<Path> MainFiles,
>> +                BackgroundIndexStorage::Factory &IndexStorageFactory,
>> +                const GlobalCompilationDatabase &CDB) {
>> +  BackgroundIndexLoader Loader;
>> +  for (llvm::StringRef MainFile : MainFiles) {
>> +    assert(llvm::sys::path::is_absolute(MainFile));
>> +
>> +    std::string ProjectRoot;
>> +    if (auto PI = CDB.getProjectInfo(MainFile))
>> +      ProjectRoot = std::move(PI->SourceRoot);
>> +    BackgroundIndexStorage *Storage = IndexStorageFactory(ProjectRoot);
>> +    Loader.load(MainFile, Storage);
>> +  }
>> +  return std::move(Loader).takeResult();
>> +}
>> +
>> +} // namespace clangd
>> +} // namespace clang
>> 
>> Added: clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h
>> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h?rev=366458&view=auto <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h?rev=366458&view=auto>
>> ==============================================================================
>> --- clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h (added)
>> +++ clang-tools-extra/trunk/clangd/index/BackgroundIndexLoader.h Thu Jul 18 09:25:36 2019
>> @@ -0,0 +1,54 @@
>> +//===--- BackgroundIndexLoader.h - Load shards from index storage-*- C++-*-===//
>> +//
>> +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
>> +// See https://llvm.org/LICENSE.txt <https://llvm.org/LICENSE.txt> for license information.
>> +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
>> +//
>> +//===----------------------------------------------------------------------===//
>> +
>> +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_INDEX_LOADER_H
>> +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_INDEX_LOADER_H
>> +
>> +#include "Path.h"
>> +#include "index/Background.h"
>> +#include "llvm/ADT/ArrayRef.h"
>> +#include "llvm/ADT/DenseMap.h"
>> +#include "llvm/ADT/Optional.h"
>> +#include "llvm/ADT/StringMap.h"
>> +#include "llvm/ADT/StringRef.h"
>> +#include "llvm/Support/VirtualFileSystem.h"
>> +#include <memory>
>> +#include <vector>
>> +
>> +namespace clang {
>> +namespace clangd {
>> +
>> +/// Represents a shard loaded from storage, stores contents in \p Shard and
>> +/// metadata about the source file that generated this shard.
>> +struct LoadedShard {
>> +  /// Path of the source file that produced this shard.
>> +  Path AbsolutePath;
>> +  /// Digest of the source file contents that produced this shard.
>> +  FileDigest Digest = {};
>> +  /// Whether the RefSlab in Shard should be used for updating symbol reference
>> +  /// counts when building an index.
>> +  bool CountReferences = false;
>> +  /// Whether the indexing action producing that shard had errors.
>> +  bool HadErrors = false;
>> +  /// Path to a TU that is depending on this shard.
>> +  Path DependentTU;
>> +  /// Will be nullptr when index storage couldn't provide a valid shard for
>> +  /// AbsolutePath.
>> +  std::unique_ptr<IndexFileIn> Shard;
>> +};
>> +
>> +/// Loads all shards for the TU \p MainFile from \p Storage.
>> +std::vector<LoadedShard>
>> +loadIndexShards(llvm::ArrayRef<Path> MainFiles,
>> +                BackgroundIndexStorage::Factory &IndexStorageFactory,
>> +                const GlobalCompilationDatabase &CDB);
>> +
>> +} // namespace clangd
>> +} // namespace clang
>> +
>> +#endif
>> 
>> Modified: clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp
>> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp?rev=366458&r1=366457&r2=366458&view=diff <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp?rev=366458&r1=366457&r2=366458&view=diff>
>> ==============================================================================
>> --- clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp (original)
>> +++ clang-tools-extra/trunk/clangd/index/BackgroundRebuild.cpp Thu Jul 18 09:25:36 2019
>> @@ -78,13 +78,13 @@ void BackgroundIndexRebuilder::idle() {
>> void BackgroundIndexRebuilder::startLoading() {
>>   std::lock_guard<std::mutex> Lock(Mu);
>>   if (!Loading)
>> -    LoadedTUs = 0;
>> +    LoadedShards = 0;
>>   ++Loading;
>> }
>> -void BackgroundIndexRebuilder::loadedTU() {
>> +void BackgroundIndexRebuilder::loadedShard(size_t ShardCount) {
>>   std::lock_guard<std::mutex> Lock(Mu);
>>   assert(Loading);
>> -  ++LoadedTUs;
>> +  LoadedShards += ShardCount;
>> }
>> void BackgroundIndexRebuilder::doneLoading() {
>>   maybeRebuild("after loading index from disk", [this] {
>> @@ -93,7 +93,7 @@ void BackgroundIndexRebuilder::doneLoadi
>>     if (Loading)    // was loading multiple batches concurrently
>>       return false; // rebuild once the last batch is done.
>>     // Rebuild if we loaded any shards, or if we stopped an indexedTU rebuild.
>> -    return LoadedTUs > 0 || enoughTUsToRebuild();
>> +    return LoadedShards > 0 || enoughTUsToRebuild();
>>   });
>> }
>> 
>> 
>> Modified: clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h
>> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h?rev=366458&r1=366457&r2=366458&view=diff <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h?rev=366458&r1=366457&r2=366458&view=diff>
>> ==============================================================================
>> --- clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h (original)
>> +++ clang-tools-extra/trunk/clangd/index/BackgroundRebuild.h Thu Jul 18 09:25:36 2019
>> @@ -17,6 +17,7 @@
>> #include "index/FileIndex.h"
>> #include "index/Index.h"
>> #include "llvm/Support/Threading.h"
>> +#include <cstddef>
>> 
>> namespace clang {
>> namespace clangd {
>> @@ -61,7 +62,7 @@ public:
>>   // sessions may happen concurrently.
>>   void startLoading();
>>   // Called to indicate some shards were actually loaded from disk.
>> -  void loadedTU();
>> +  void loadedShard(size_t ShardCount);
>>   // Called to indicate we're finished loading shards from disk.
>>   // May rebuild (if any were loaded).
>>   void doneLoading();
>> @@ -89,7 +90,7 @@ private:
>>   unsigned IndexedTUsAtLastRebuild = 0;
>>   // Are we loading shards? May be multiple concurrent sessions.
>>   unsigned Loading = 0;
>> -  unsigned LoadedTUs; // In the current loading session.
>> +  unsigned LoadedShards; // In the current loading session.
>> 
>>   SwapIndex *Target;
>>   FileSymbols *Source;
>> 
>> Modified: clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp
>> URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp?rev=366458&r1=366457&r2=366458&view=diff <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp?rev=366458&r1=366457&r2=366458&view=diff>
>> ==============================================================================
>> --- clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp (original)
>> +++ clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp Thu Jul 18 09:25:36 2019
>> @@ -211,7 +211,7 @@ TEST_F(BackgroundIndexTest, ShardStorage
>>     OverlayCDB CDB(/*Base=*/nullptr);
>>     BackgroundIndex Idx(Context::empty(), FS, CDB,
>>                         [&](llvm::StringRef) { return &MSS; });
>> -    CDB.setCompileCommand(testPath("root"), Cmd);
>> +    CDB.setCompileCommand(testPath("root/A.cc <http://a.cc/>"), Cmd);
>>     ASSERT_TRUE(Idx.blockUntilIdleForTest());
>>   }
>>   EXPECT_EQ(CacheHits, 2U); // Check both A.cc <http://a.cc/> and A.h loaded from cache.
>> @@ -335,7 +335,7 @@ TEST_F(BackgroundIndexTest, ShardStorage
>>     OverlayCDB CDB(/*Base=*/nullptr);
>>     BackgroundIndex Idx(Context::empty(), FS, CDB,
>>                         [&](llvm::StringRef) { return &MSS; });
>> -    CDB.setCompileCommand(testPath("root"), Cmd);
>> +    CDB.setCompileCommand(testPath("root/A.cc <http://a.cc/>"), Cmd);
>>     ASSERT_TRUE(Idx.blockUntilIdleForTest());
>>   }
>>   EXPECT_EQ(CacheHits, 2U); // Check both A.cc <http://a.cc/> and A.h loaded from cache.
>> @@ -353,7 +353,7 @@ TEST_F(BackgroundIndexTest, ShardStorage
>>     OverlayCDB CDB(/*Base=*/nullptr);
>>     BackgroundIndex Idx(Context::empty(), FS, CDB,
>>                         [&](llvm::StringRef) { return &MSS; });
>> -    CDB.setCompileCommand(testPath("root"), Cmd);
>> +    CDB.setCompileCommand(testPath("root/A.cc <http://a.cc/>"), Cmd);
>>     ASSERT_TRUE(Idx.blockUntilIdleForTest());
>>   }
>>   EXPECT_EQ(CacheHits, 2U); // Check both A.cc <http://a.cc/> and A.h loaded from cache.
>> @@ -621,8 +621,8 @@ TEST_F(BackgroundIndexRebuilderTest, Ind
>> 
>> TEST_F(BackgroundIndexRebuilderTest, LoadingShards) {
>>   Rebuilder.startLoading();
>> -  Rebuilder.loadedTU();
>> -  Rebuilder.loadedTU();
>> +  Rebuilder.loadedShard(10);
>> +  Rebuilder.loadedShard(20);
>>   EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
>> 
>>   // No rebuild for no shards.
>> @@ -631,11 +631,11 @@ TEST_F(BackgroundIndexRebuilderTest, Loa
>> 
>>   // Loads can overlap.
>>   Rebuilder.startLoading();
>> -  Rebuilder.loadedTU();
>> +  Rebuilder.loadedShard(1);
>>   Rebuilder.startLoading();
>> -  Rebuilder.loadedTU();
>> +  Rebuilder.loadedShard(1);
>>   EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
>> -  Rebuilder.loadedTU();
>> +  Rebuilder.loadedShard(1);
>>   EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
>> 
>>   // No rebuilding for indexed files while loading.
>> 
>> 
>> _______________________________________________
>> cfe-commits mailing list
>> cfe-commits at lists.llvm.org <mailto:cfe-commits at lists.llvm.org>
>> https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
> 

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20190719/3baf91e1/attachment-0001.html>


More information about the cfe-commits mailing list