[clang-tools-extra] r347235 - [clangd] Store source file hash in IndexFile{In, Out}

Kadir Cetinkaya via cfe-commits cfe-commits at lists.llvm.org
Mon Nov 19 10:06:30 PST 2018


Author: kadircet
Date: Mon Nov 19 10:06:29 2018
New Revision: 347235

URL: http://llvm.org/viewvc/llvm-project?rev=347235&view=rev
Log:
[clangd] Store source file hash in IndexFile{In,Out}

Summary:
Puts the digest of the source file that generated the index into
serialized index and stores them back on load, if exists.

Reviewers: sammccall

Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, arphaman, cfe-commits

Differential Revision: https://reviews.llvm.org/D54693

Modified:
    clang-tools-extra/trunk/clangd/index/Background.cpp
    clang-tools-extra/trunk/clangd/index/Serialization.cpp
    clang-tools-extra/trunk/clangd/index/Serialization.h
    clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp
    clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/Background.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.cpp?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Background.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Background.cpp Mon Nov 19 10:06:29 2018
@@ -246,6 +246,7 @@ void BackgroundIndex::update(StringRef M
       IndexFileOut Shard;
       Shard.Symbols = SS.get();
       Shard.Refs = RS.get();
+      Shard.Digest = &Hash;
       if (auto Error = IndexStorage->storeShard(Path, Shard))
         elog("Failed to write background-index shard for file {0}: {1}", Path,
              std::move(Error));

Modified: clang-tools-extra/trunk/clangd/index/Serialization.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Serialization.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Serialization.cpp Mon Nov 19 10:06:29 2018
@@ -363,6 +363,13 @@ Expected<IndexFileIn> readRIFF(StringRef
     return Strings.takeError();
 
   IndexFileIn Result;
+  if (Chunks.count("hash")) {
+    Reader Hash(Chunks.lookup("hash"));
+    llvm::StringRef Digest = Hash.consume(20);
+    Result.Digest.emplace();
+    std::copy(Digest.bytes_begin(), Digest.bytes_end(), Result.Digest->begin());
+  }
+
   if (Chunks.count("symb")) {
     Reader SymbolReader(Chunks.lookup("symb"));
     SymbolSlab::Builder Symbols;
@@ -399,6 +406,12 @@ void writeRIFF(const IndexFileOut &Data,
   }
   RIFF.Chunks.push_back({riff::fourCC("meta"), Meta});
 
+  if (Data.Digest) {
+    llvm::StringRef Hash(reinterpret_cast<const char *>(Data.Digest->data()),
+                         Data.Digest->size());
+    RIFF.Chunks.push_back({riff::fourCC("hash"), Hash});
+  }
+
   StringTableOut Strings;
   std::vector<Symbol> Symbols;
   for (const auto &Sym : *Data.Symbols) {

Modified: clang-tools-extra/trunk/clangd/index/Serialization.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.h?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Serialization.h (original)
+++ clang-tools-extra/trunk/clangd/index/Serialization.h Mon Nov 19 10:06:29 2018
@@ -39,6 +39,8 @@ enum class IndexFileFormat {
 struct IndexFileIn {
   llvm::Optional<SymbolSlab> Symbols;
   llvm::Optional<RefSlab> Refs;
+  // Digest of the source file that generated the contents.
+  llvm::Optional<std::array<uint8_t, 20>> Digest;
 };
 // Parse an index file. The input must be a RIFF or YAML file.
 llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
@@ -47,6 +49,8 @@ llvm::Expected<IndexFileIn> readIndexFil
 struct IndexFileOut {
   const SymbolSlab *Symbols = nullptr;
   const RefSlab *Refs = nullptr;
+  // Digest of the source file that generated the contents.
+  const std::array<uint8_t, 20> *Digest = nullptr;
   // TODO: Support serializing Dex posting lists.
   IndexFileFormat Format = IndexFileFormat::RIFF;
 

Modified: clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp Mon Nov 19 10:06:29 2018
@@ -121,8 +121,10 @@ TEST(BackgroundIndexTest, ShardStorageWr
       void f_b();
       class A_CC {};
       )cpp";
-  FS.Files[testPath("root/A.cc")] =
-      "#include \"A.h\"\nvoid g() { (void)common; }";
+  std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }";
+  FS.Files[testPath("root/A.cc")] = A_CC;
+  auto Digest = llvm::SHA1::hash(
+      {reinterpret_cast<const uint8_t *>(A_CC.data()), A_CC.size()});
 
   llvm::StringMap<std::string> Storage;
   size_t CacheHits = 0;
@@ -156,6 +158,7 @@ TEST(BackgroundIndexTest, ShardStorageWr
   EXPECT_NE(ShardSource, nullptr);
   EXPECT_THAT(*ShardSource->Symbols, UnorderedElementsAre());
   EXPECT_THAT(*ShardSource->Refs, RefsAre({FileURI("unittest:///root/A.cc")}));
+  EXPECT_EQ(*ShardSource->Digest, Digest);
 }
 
 } // namespace clangd

Modified: clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp Mon Nov 19 10:06:29 2018
@@ -9,6 +9,7 @@
 
 #include "index/Index.h"
 #include "index/Serialization.h"
+#include "llvm/Support/SHA1.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
@@ -165,6 +166,33 @@ TEST(SerializationTest, BinaryConversion
   ASSERT_TRUE(In2->Symbols);
   ASSERT_TRUE(In2->Refs);
 
+  // Assert the YAML serializations match, for nice comparisons and diffs.
+  EXPECT_THAT(YAMLFromSymbols(*In2->Symbols),
+              UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols)));
+  EXPECT_THAT(YAMLFromRefs(*In2->Refs),
+              UnorderedElementsAreArray(YAMLFromRefs(*In->Refs)));
+}
+
+TEST(SerializationTest, HashTest) {
+  auto In = readIndexFile(YAML);
+  EXPECT_TRUE(bool(In)) << In.takeError();
+
+  std::string TestContent("TESTCONTENT");
+  auto Digest =
+      llvm::SHA1::hash({reinterpret_cast<const uint8_t *>(TestContent.data()),
+                        TestContent.size()});
+  // Write to binary format, and parse again.
+  IndexFileOut Out(*In);
+  Out.Format = IndexFileFormat::RIFF;
+  Out.Digest = &Digest;
+  std::string Serialized = to_string(Out);
+
+  auto In2 = readIndexFile(Serialized);
+  ASSERT_TRUE(bool(In2)) << In.takeError();
+  ASSERT_EQ(In2->Digest, Digest);
+  ASSERT_TRUE(In2->Symbols);
+  ASSERT_TRUE(In2->Refs);
+
   // Assert the YAML serializations match, for nice comparisons and diffs.
   EXPECT_THAT(YAMLFromSymbols(*In2->Symbols),
               UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols)));




More information about the cfe-commits mailing list