[clang-tools-extra] r347235 - [clangd] Store source file hash in IndexFile{In, Out}
Kadir Cetinkaya via cfe-commits
cfe-commits at lists.llvm.org
Mon Nov 19 10:06:30 PST 2018
Author: kadircet
Date: Mon Nov 19 10:06:29 2018
New Revision: 347235
URL: http://llvm.org/viewvc/llvm-project?rev=347235&view=rev
Log:
[clangd] Store source file hash in IndexFile{In,Out}
Summary:
Puts the digest of the source file that generated the index into
serialized index and stores them back on load, if exists.
Reviewers: sammccall
Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, arphaman, cfe-commits
Differential Revision: https://reviews.llvm.org/D54693
Modified:
clang-tools-extra/trunk/clangd/index/Background.cpp
clang-tools-extra/trunk/clangd/index/Serialization.cpp
clang-tools-extra/trunk/clangd/index/Serialization.h
clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp
clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp
Modified: clang-tools-extra/trunk/clangd/index/Background.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.cpp?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Background.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Background.cpp Mon Nov 19 10:06:29 2018
@@ -246,6 +246,7 @@ void BackgroundIndex::update(StringRef M
IndexFileOut Shard;
Shard.Symbols = SS.get();
Shard.Refs = RS.get();
+ Shard.Digest = &Hash;
if (auto Error = IndexStorage->storeShard(Path, Shard))
elog("Failed to write background-index shard for file {0}: {1}", Path,
std::move(Error));
Modified: clang-tools-extra/trunk/clangd/index/Serialization.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Serialization.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Serialization.cpp Mon Nov 19 10:06:29 2018
@@ -363,6 +363,13 @@ Expected<IndexFileIn> readRIFF(StringRef
return Strings.takeError();
IndexFileIn Result;
+ if (Chunks.count("hash")) {
+ Reader Hash(Chunks.lookup("hash"));
+ llvm::StringRef Digest = Hash.consume(20);
+ Result.Digest.emplace();
+ std::copy(Digest.bytes_begin(), Digest.bytes_end(), Result.Digest->begin());
+ }
+
if (Chunks.count("symb")) {
Reader SymbolReader(Chunks.lookup("symb"));
SymbolSlab::Builder Symbols;
@@ -399,6 +406,12 @@ void writeRIFF(const IndexFileOut &Data,
}
RIFF.Chunks.push_back({riff::fourCC("meta"), Meta});
+ if (Data.Digest) {
+ llvm::StringRef Hash(reinterpret_cast<const char *>(Data.Digest->data()),
+ Data.Digest->size());
+ RIFF.Chunks.push_back({riff::fourCC("hash"), Hash});
+ }
+
StringTableOut Strings;
std::vector<Symbol> Symbols;
for (const auto &Sym : *Data.Symbols) {
Modified: clang-tools-extra/trunk/clangd/index/Serialization.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.h?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Serialization.h (original)
+++ clang-tools-extra/trunk/clangd/index/Serialization.h Mon Nov 19 10:06:29 2018
@@ -39,6 +39,8 @@ enum class IndexFileFormat {
struct IndexFileIn {
llvm::Optional<SymbolSlab> Symbols;
llvm::Optional<RefSlab> Refs;
+ // Digest of the source file that generated the contents.
+ llvm::Optional<std::array<uint8_t, 20>> Digest;
};
// Parse an index file. The input must be a RIFF or YAML file.
llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
@@ -47,6 +49,8 @@ llvm::Expected<IndexFileIn> readIndexFil
struct IndexFileOut {
const SymbolSlab *Symbols = nullptr;
const RefSlab *Refs = nullptr;
+ // Digest of the source file that generated the contents.
+ const std::array<uint8_t, 20> *Digest = nullptr;
// TODO: Support serializing Dex posting lists.
IndexFileFormat Format = IndexFileFormat::RIFF;
Modified: clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp Mon Nov 19 10:06:29 2018
@@ -121,8 +121,10 @@ TEST(BackgroundIndexTest, ShardStorageWr
void f_b();
class A_CC {};
)cpp";
- FS.Files[testPath("root/A.cc")] =
- "#include \"A.h\"\nvoid g() { (void)common; }";
+ std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }";
+ FS.Files[testPath("root/A.cc")] = A_CC;
+ auto Digest = llvm::SHA1::hash(
+ {reinterpret_cast<const uint8_t *>(A_CC.data()), A_CC.size()});
llvm::StringMap<std::string> Storage;
size_t CacheHits = 0;
@@ -156,6 +158,7 @@ TEST(BackgroundIndexTest, ShardStorageWr
EXPECT_NE(ShardSource, nullptr);
EXPECT_THAT(*ShardSource->Symbols, UnorderedElementsAre());
EXPECT_THAT(*ShardSource->Refs, RefsAre({FileURI("unittest:///root/A.cc")}));
+ EXPECT_EQ(*ShardSource->Digest, Digest);
}
} // namespace clangd
Modified: clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp?rev=347235&r1=347234&r2=347235&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp Mon Nov 19 10:06:29 2018
@@ -9,6 +9,7 @@
#include "index/Index.h"
#include "index/Serialization.h"
+#include "llvm/Support/SHA1.h"
#include "llvm/Support/ScopedPrinter.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
@@ -165,6 +166,33 @@ TEST(SerializationTest, BinaryConversion
ASSERT_TRUE(In2->Symbols);
ASSERT_TRUE(In2->Refs);
+ // Assert the YAML serializations match, for nice comparisons and diffs.
+ EXPECT_THAT(YAMLFromSymbols(*In2->Symbols),
+ UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols)));
+ EXPECT_THAT(YAMLFromRefs(*In2->Refs),
+ UnorderedElementsAreArray(YAMLFromRefs(*In->Refs)));
+}
+
+TEST(SerializationTest, HashTest) {
+ auto In = readIndexFile(YAML);
+ EXPECT_TRUE(bool(In)) << In.takeError();
+
+ std::string TestContent("TESTCONTENT");
+ auto Digest =
+ llvm::SHA1::hash({reinterpret_cast<const uint8_t *>(TestContent.data()),
+ TestContent.size()});
+ // Write to binary format, and parse again.
+ IndexFileOut Out(*In);
+ Out.Format = IndexFileFormat::RIFF;
+ Out.Digest = &Digest;
+ std::string Serialized = to_string(Out);
+
+ auto In2 = readIndexFile(Serialized);
+ ASSERT_TRUE(bool(In2)) << In.takeError();
+ ASSERT_EQ(In2->Digest, Digest);
+ ASSERT_TRUE(In2->Symbols);
+ ASSERT_TRUE(In2->Refs);
+
// Assert the YAML serializations match, for nice comparisons and diffs.
EXPECT_THAT(YAMLFromSymbols(*In2->Symbols),
UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols)));
More information about the cfe-commits
mailing list