[clang-tools-extra] r365121 - [clangd] Store hash of command line in index shards.
Kadir Cetinkaya via cfe-commits
cfe-commits at lists.llvm.org
Thu Jul 4 02:51:53 PDT 2019
Author: kadircet
Date: Thu Jul 4 02:51:53 2019
New Revision: 365121
URL: http://llvm.org/viewvc/llvm-project?rev=365121&view=rev
Log:
[clangd] Store hash of command line in index shards.
Summary: This is to enable cache invalidation when command line flags changes.
Reviewers: sammccall
Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D64018
Modified:
clang-tools-extra/trunk/clangd/index/Background.cpp
clang-tools-extra/trunk/clangd/index/Serialization.cpp
clang-tools-extra/trunk/clangd/index/Serialization.h
clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp
clang-tools-extra/trunk/clangd/unittests/SerializationTests.cpp
Modified: clang-tools-extra/trunk/clangd/index/Background.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.cpp?rev=365121&r1=365120&r2=365121&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Background.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Background.cpp Thu Jul 4 02:51:53 2019
@@ -25,6 +25,7 @@
#include "index/SymbolCollector.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/StringMap.h"
@@ -373,6 +374,11 @@ void BackgroundIndex::update(llvm::Strin
Shard.Relations = RelS.get();
Shard.Sources = IG.get();
+ // Only store command line hash for main files of the TU, since our
+ // current model keeps only one version of a header file.
+ if (Path == MainFile)
+ Shard.Cmd = Index.Cmd.getPointer();
+
if (auto Error = IndexStorage->storeShard(Path, Shard))
elog("Failed to write background-index shard for file {0}: {1}", Path,
std::move(Error));
@@ -479,6 +485,7 @@ llvm::Error BackgroundIndex::index(tooli
Action->EndSourceFile();
+ Index.Cmd = Inputs.CompileCommand;
assert(Index.Symbols && Index.Refs && Index.Sources &&
"Symbols, Refs and Sources must be set.");
Modified: clang-tools-extra/trunk/clangd/index/Serialization.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=365121&r1=365120&r2=365121&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Serialization.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Serialization.cpp Thu Jul 4 02:51:53 2019
@@ -13,9 +13,13 @@
#include "SymbolOrigin.h"
#include "Trace.h"
#include "dex/Dex.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Compression.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+#include <vector>
namespace clang {
namespace clangd {
@@ -403,6 +407,30 @@ Relation readRelation(Reader &Data) {
return {Subject, Predicate, Object};
}
+struct InternedCompileCommand {
+ llvm::StringRef Directory;
+ std::vector<llvm::StringRef> CommandLine;
+};
+
+void writeCompileCommand(const InternedCompileCommand &Cmd,
+ const StringTableOut &Strings,
+ llvm::raw_ostream &CmdOS) {
+ writeVar(Strings.index(Cmd.Directory), CmdOS);
+ writeVar(Cmd.CommandLine.size(), CmdOS);
+ for (llvm::StringRef C : Cmd.CommandLine)
+ writeVar(Strings.index(C), CmdOS);
+}
+
+InternedCompileCommand
+readCompileCommand(Reader CmdReader, llvm::ArrayRef<llvm::StringRef> Strings) {
+ InternedCompileCommand Cmd;
+ Cmd.Directory = CmdReader.consumeString(Strings);
+ Cmd.CommandLine.resize(CmdReader.consumeVar());
+ for (llvm::StringRef &C : Cmd.CommandLine)
+ C = CmdReader.consumeString(Strings);
+ return Cmd;
+}
+
// FILE ENCODING
// A file is a RIFF chunk with type 'CdIx'.
// It contains the sections:
@@ -490,6 +518,18 @@ llvm::Expected<IndexFileIn> readRIFF(llv
return makeError("malformed or truncated relations");
Result.Relations = std::move(Relations).build();
}
+ if (Chunks.count("cmdl")) {
+ Reader CmdReader(Chunks.lookup("cmdl"));
+ if (CmdReader.err())
+ return makeError("malformed or truncated commandline section");
+ InternedCompileCommand Cmd =
+ readCompileCommand(CmdReader, Strings->Strings);
+ Result.Cmd.emplace();
+ Result.Cmd->Directory = Cmd.Directory;
+ Result.Cmd->CommandLine.reserve(Cmd.CommandLine.size());
+ for (llvm::StringRef C : Cmd.CommandLine)
+ Result.Cmd->CommandLine.emplace_back(C);
+ }
return std::move(Result);
}
@@ -547,6 +587,17 @@ void writeRIFF(const IndexFileOut &Data,
}
}
+ InternedCompileCommand InternedCmd;
+ if (Data.Cmd) {
+ InternedCmd.CommandLine.reserve(Data.Cmd->CommandLine.size());
+ InternedCmd.Directory = Data.Cmd->Directory;
+ Strings.intern(InternedCmd.Directory);
+ for (llvm::StringRef C : Data.Cmd->CommandLine) {
+ InternedCmd.CommandLine.emplace_back(C);
+ Strings.intern(InternedCmd.CommandLine.back());
+ }
+ }
+
std::string StringSection;
{
llvm::raw_string_ostream StringOS(StringSection);
@@ -592,6 +643,15 @@ void writeRIFF(const IndexFileOut &Data,
RIFF.Chunks.push_back({riff::fourCC("srcs"), SrcsSection});
}
+ std::string CmdlSection;
+ if (Data.Cmd) {
+ {
+ llvm::raw_string_ostream CmdOS(CmdlSection);
+ writeCompileCommand(InternedCmd, Strings, CmdOS);
+ }
+ RIFF.Chunks.push_back({riff::fourCC("cmdl"), CmdlSection});
+ }
+
OS << RIFF;
}
Modified: clang-tools-extra/trunk/clangd/index/Serialization.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.h?rev=365121&r1=365120&r2=365121&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Serialization.h (original)
+++ clang-tools-extra/trunk/clangd/index/Serialization.h Thu Jul 4 02:51:53 2019
@@ -27,6 +27,7 @@
#include "Headers.h"
#include "Index.h"
#include "index/Symbol.h"
+#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/Support/Error.h"
namespace clang {
@@ -44,6 +45,8 @@ struct IndexFileIn {
llvm::Optional<RelationSlab> Relations;
// Keys are URIs of the source files.
llvm::Optional<IncludeGraph> Sources;
+ // This contains only the Directory and CommandLine.
+ llvm::Optional<tooling::CompileCommand> Cmd;
};
// Parse an index file. The input must be a RIFF or YAML file.
llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
@@ -57,13 +60,15 @@ struct IndexFileOut {
const IncludeGraph *Sources = nullptr;
// TODO: Support serializing Dex posting lists.
IndexFileFormat Format = IndexFileFormat::RIFF;
+ const tooling::CompileCommand *Cmd = nullptr;
IndexFileOut() = default;
IndexFileOut(const IndexFileIn &I)
: Symbols(I.Symbols ? I.Symbols.getPointer() : nullptr),
Refs(I.Refs ? I.Refs.getPointer() : nullptr),
Relations(I.Relations ? I.Relations.getPointer() : nullptr),
- Sources(I.Sources ? I.Sources.getPointer() : nullptr) {}
+ Sources(I.Sources ? I.Sources.getPointer() : nullptr),
+ Cmd(I.Cmd ? I.Cmd.getPointer() : nullptr) {}
};
// Serializes an index file.
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O);
Modified: clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp?rev=365121&r1=365120&r2=365121&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp (original)
+++ clang-tools-extra/trunk/clangd/unittests/BackgroundIndexTests.cpp Thu Jul 4 02:51:53 2019
@@ -2,6 +2,7 @@
#include "TestFS.h"
#include "TestTU.h"
#include "index/Background.h"
+#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/Support/ScopedPrinter.h"
#include "llvm/Support/Threading.h"
#include "gmock/gmock.h"
@@ -526,5 +527,49 @@ TEST_F(BackgroundIndexTest, Uncompilable
"unittest:///B.h"));
}
+TEST_F(BackgroundIndexTest, CmdLineHash) {
+ MockFSProvider FS;
+ llvm::StringMap<std::string> Storage;
+ size_t CacheHits = 0;
+ MemoryShardStorage MSS(Storage, CacheHits);
+ OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{},
+ /*ResourceDir=*/std::string(""));
+ BackgroundIndex Idx(Context::empty(), FS, CDB,
+ [&](llvm::StringRef) { return &MSS; });
+
+ tooling::CompileCommand Cmd;
+ FS.Files[testPath("A.cc")] = "#include \"A.h\"";
+ FS.Files[testPath("A.h")] = "";
+ Cmd.Filename = "../A.cc";
+ Cmd.Directory = testPath("build");
+ Cmd.CommandLine = {"clang++", "../A.cc", "-fsyntax-only"};
+ CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
+ ASSERT_TRUE(Idx.blockUntilIdleForTest());
+
+ EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h")));
+ // Make sure we only store the Cmd for main file.
+ EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
+
+ {
+ tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
+ EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
+ EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
+ }
+
+ // FIXME: Changing compile commands should be enough to invalidate the cache.
+ FS.Files[testPath("A.cc")] = " ";
+ Cmd.CommandLine = {"clang++", "../A.cc", "-Dfoo", "-fsyntax-only"};
+ CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
+ ASSERT_TRUE(Idx.blockUntilIdleForTest());
+
+ EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
+
+ {
+ tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
+ EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
+ EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
+ }
+}
+
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/unittests/SerializationTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/SerializationTests.cpp?rev=365121&r1=365120&r2=365121&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/unittests/SerializationTests.cpp (original)
+++ clang-tools-extra/trunk/clangd/unittests/SerializationTests.cpp Thu Jul 4 02:51:53 2019
@@ -8,6 +8,7 @@
#include "index/Index.h"
#include "index/Serialization.h"
+#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/Support/SHA1.h"
#include "llvm/Support/ScopedPrinter.h"
#include "gmock/gmock.h"
@@ -240,6 +241,36 @@ TEST(SerializationTest, SrcsTest) {
}
}
+TEST(SerializationTest, CmdlTest) {
+ auto In = readIndexFile(YAML);
+ EXPECT_TRUE(bool(In)) << In.takeError();
+
+ tooling::CompileCommand Cmd;
+ Cmd.Directory = "testdir";
+ Cmd.CommandLine.push_back("cmd1");
+ Cmd.CommandLine.push_back("cmd2");
+ Cmd.Filename = "ignored";
+ Cmd.Heuristic = "ignored";
+ Cmd.Output = "ignored";
+
+ IndexFileOut Out(*In);
+ Out.Format = IndexFileFormat::RIFF;
+ Out.Cmd = &Cmd;
+ {
+ std::string Serialized = llvm::to_string(Out);
+
+ auto In = readIndexFile(Serialized);
+ ASSERT_TRUE(bool(In)) << In.takeError();
+ ASSERT_TRUE(In->Cmd);
+
+ const tooling::CompileCommand &SerializedCmd = In->Cmd.getValue();
+ EXPECT_EQ(SerializedCmd.CommandLine, Cmd.CommandLine);
+ EXPECT_EQ(SerializedCmd.Directory, Cmd.Directory);
+ EXPECT_NE(SerializedCmd.Filename, Cmd.Filename);
+ EXPECT_NE(SerializedCmd.Heuristic, Cmd.Heuristic);
+ EXPECT_NE(SerializedCmd.Output, Cmd.Output);
+ }
+}
} // namespace
} // namespace clangd
} // namespace clang
More information about the cfe-commits
mailing list