<div dir="ltr"><div dir="ltr"><div dir="ltr"><div dir="ltr"><div dir="ltr">Thanks. Unclear to me whether it's the enum class or the anonymous namespace that's triggering this (I believe) compiler bug, but r341459 may help...</div></div></div></div></div><br><div class="gmail_quote"><div dir="ltr">On Wed, Sep 5, 2018 at 11:05 AM Mikael Holmén <<a href="mailto:mikael.holmen@ericsson.com">mikael.holmen@ericsson.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"><br>
<br>
On 09/05/2018 09:56 AM, Sam McCall wrote:<br>
> Sorry! r341451 should fix this, will keep an eye on the buildbots.<br>
> <br>
<br>
Now it compiles with clang 3.6.0 but with gcc 5.4.0 it fails with<br>
<br>
/proj/bbi_twh/wh_bbi/x86_64-Linux2/bbigcc/<a href="http://1.5.4.0/crosscompiler/bin/g++" rel="noreferrer" target="_blank">1.5.4.0/crosscompiler/bin/g++</a> <br>
-I/proj/bbi_twh/wh_bbi/x86_64-Linux2/bbilibxml2/1/include <br>
-DGTEST_HAS_RTTI=0 -D_DEBUG -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS <br>
-D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS <br>
-Itools/clang/tools/extra/clangd/global-symbol-builder <br>
-I../tools/clang/tools/extra/clangd/global-symbol-builder <br>
-I../tools/clang/include -Itools/clang/include -I/usr/include/libxml2 <br>
-Iinclude -I../include <br>
-I../tools/clang/tools/extra/clangd/global-symbol-builder/.. <br>
-I/repo/app/valgrind/3.11.0/include -fPIC -fvisibility-inlines-hidden <br>
-Werror=date-time -std=c++11 -Wall -Wextra -Wno-unused-parameter <br>
-Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic <br>
-Wno-long-long -Wno-maybe-uninitialized -Wdelete-non-virtual-dtor <br>
-Wno-comment -fdiagnostics-color -ffunction-sections -fdata-sections <br>
-fno-common -Woverloaded-virtual -fno-strict-aliasing -O3 -UNDEBUG <br>
-fno-exceptions -fno-rtti -MMD -MT <br>
tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o <br>
-MF <br>
tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o.d <br>
-o <br>
tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o <br>
-c <br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
In file included from <br>
../tools/clang/include/clang/Frontend/CommandLineSourceLoc.h:19:0,<br>
from <br>
../tools/clang/include/clang/Frontend/FrontendOptions.h:13,<br>
from <br>
../tools/clang/include/clang/Frontend/CompilerInvocation.h:19,<br>
from <br>
../tools/clang/include/clang/Frontend/CompilerInstance.h:16,<br>
from <br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:22:<br>
../include/llvm/Support/CommandLine.h:606:52: error: invalid cast from <br>
type 'llvm::cl::opt<clang::clangd::{anonymous}::Format>' to type 'int'<br>
llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:67:16: <br>
note: in expansion of macro 'clEnumValN'<br>
clEnumValN(Format::YAML, "yaml", "human-readable YAML <br>
format"),<br>
^<br>
../include/llvm/Support/CommandLine.h:606:29: error: expected <br>
primary-expression before '{' token<br>
llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:67:16: <br>
note: in expansion of macro 'clEnumValN'<br>
clEnumValN(Format::YAML, "yaml", "human-readable YAML <br>
format"),<br>
^<br>
../include/llvm/Support/CommandLine.h:606:52: error: invalid cast from <br>
type 'llvm::cl::opt<clang::clangd::{anonymous}::Format>' to type 'int'<br>
llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:68:16: <br>
note: in expansion of macro 'clEnumValN'<br>
clEnumValN(Format::Binary, "binary", "binary RIFF <br>
format")),<br>
^<br>
../include/llvm/Support/CommandLine.h:606:29: error: expected <br>
primary-expression before '{' token<br>
llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:68:16: <br>
note: in expansion of macro 'clEnumValN'<br>
clEnumValN(Format::Binary, "binary", "binary RIFF <br>
format")),<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:69:27: <br>
error: 'Format' is not a class, namespace, or enumeration<br>
llvm::cl::init(Format::YAML));<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp: <br>
In function 'int main(int, const char**)':<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:276:23: <br>
error: 'clang::clangd::Format' is not a class, namespace, or enumeration<br>
case clang::clangd::Format::YAML:<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:279:23: <br>
error: 'clang::clangd::Format' is not a class, namespace, or enumeration<br>
case clang::clangd::Format::Binary: {<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10: <br>
warning: enumeration value 'YAML' not handled in switch [-Wswitch]<br>
switch (clang::clangd::Format) {<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10: <br>
warning: enumeration value 'Binary' not handled in switch [-Wswitch]<br>
<br>
/Mikael<br>
<br>
> On Wed, Sep 5, 2018 at 8:46 AM Mikael Holmén <<a href="mailto:mikael.holmen@ericsson.com" target="_blank">mikael.holmen@ericsson.com</a> <br>
> <mailto:<a href="mailto:mikael.holmen@ericsson.com" target="_blank">mikael.holmen@ericsson.com</a>>> wrote:<br>
> <br>
> Hi Sam,<br>
> <br>
> This doesn't compile for me. Both clang 3.6.0 and gcc 5.4.0 complain:<br>
> <br>
> [1/6] Building CXX object<br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o<br>
> FAILED:<br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o<br>
> <br>
> <br>
> /usr/bin/clang++ -march=corei7 -DGTEST_HAS_RTTI=0 -D_DEBUG<br>
> -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS<br>
> -D__STDC_LIMIT_MACROS -Itools/clang/tools/extra/clangd<br>
> -I../tools/clang/tools/extra/clangd -I../tools/clang/include<br>
> -Itools/clang/include -I/usr/include/libxml2 -Iinclude -I../include<br>
> -I/proj/flexasic/app/valgrind/3.11.0/include -fPIC<br>
> -fvisibility-inlines-hidden -Werror -Werror=date-time -std=c++11 -Wall<br>
> -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual<br>
> -Wmissing-field-initializers -pedantic -Wno-long-long<br>
> -Wcovered-switch-default -Wnon-virtual-dtor -Wdelete-non-virtual-dtor<br>
> -Wstring-conversion -fdiagnostics-color -ffunction-sections<br>
> -fdata-sections -fno-common -Woverloaded-virtual -Wno-nested-anon-types<br>
> -O3 -UNDEBUG -fno-exceptions -fno-rtti -MMD -MT<br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o<br>
> <br>
> -MF<br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o.d<br>
> <br>
> -o<br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o<br>
> <br>
> -c ../tools/clang/tools/extra/clangd/index/Serialization.cpp<br>
> ../tools/clang/tools/extra/clangd/index/Serialization.cpp:154:10:<br>
> error:<br>
> no viable conversion from 'clang::clangd::(anonymous<br>
> namespace)::StringTableIn' to 'Expected<clang::clangd::(anonymous<br>
> namespace)::StringTableIn>'<br>
> return Table;<br>
> ^~~~~<br>
> ../include/llvm/Support/Error.h:434:41: note: candidate constructor<br>
> (the<br>
> implicit copy constructor) not viable: no known conversion from<br>
> 'clang::clangd::(anonymous namespace)::StringTableIn' to 'const<br>
> llvm::Expected<clang::clangd::(anonymous namespace)::StringTableIn> &'<br>
> for 1st argument<br>
> template <class T> class LLVM_NODISCARD Expected {<br>
> ^<br>
> ../include/llvm/Support/Error.h:456:3: note: candidate constructor not<br>
> viable: no known conversion from 'clang::clangd::(anonymous<br>
> namespace)::StringTableIn' to 'llvm::Error' for 1st argument<br>
> Expected(Error Err)<br>
> ^<br>
> ../include/llvm/Support/Error.h:470:3: note: candidate constructor not<br>
> viable: no known conversion from 'clang::clangd::(anonymous<br>
> namespace)::StringTableIn' to 'llvm::ErrorSuccess' for 1st argument<br>
> Expected(ErrorSuccess) = delete;<br>
> ^<br>
> ../include/llvm/Support/Error.h:488:3: note: candidate constructor not<br>
> viable: no known conversion from 'clang::clangd::(anonymous<br>
> namespace)::StringTableIn' to 'llvm::Expected<clang::clangd::(anonymous<br>
> namespace)::StringTableIn> &&' for 1st argument<br>
> Expected(Expected &&Other) { moveConstruct(std::move(Other)); }<br>
> ^<br>
> ../include/llvm/Support/Error.h:476:36: note: candidate template<br>
> ignored: disabled by 'enable_if' [with OtherT =<br>
> clang::clangd::(anonymous namespace)::StringTableIn &]<br>
> typename std::enable_if<std::is_convertible<OtherT,<br>
> T>::value>::type<br>
> ^<br>
> ../include/llvm/Support/Error.h:493:3: note: candidate template<br>
> ignored:<br>
> could not match 'Expected<type-parameter-0-0>' against<br>
> 'clang::clangd::(anonymous namespace)::StringTableIn'<br>
> Expected(Expected<OtherT> &&Other,<br>
> ^<br>
> In file included from<br>
> ../tools/clang/tools/extra/clangd/index/Serialization.cpp:9:<br>
> In file included from<br>
> ../tools/clang/tools/extra/clangd/index/Serialization.h:23:<br>
> In file included from<br>
> ../tools/clang/tools/extra/clangd/index/Index.h:13:<br>
> In file included from<br>
> ../tools/clang/include/clang/Index/IndexSymbol.h:14:<br>
> In file included from ../tools/clang/include/clang/Lex/MacroInfo.h:18:<br>
> In file included from ../tools/clang/include/clang/Lex/Token.h:17:<br>
> In file included from<br>
> ../tools/clang/include/clang/Basic/SourceLocation.h:19:<br>
> In file included from ../include/llvm/ADT/StringRef.h:13:<br>
> In file included from ../include/llvm/ADT/STLExtras.h:20:<br>
> ../include/llvm/ADT/Optional.h:41:28: error: call to implicitly-deleted<br>
> copy constructor of 'clang::clangd::SymbolSlab'<br>
> new (storage.buffer) T(*O.getPointer());<br>
> ^ ~~~~~~~~~~~~~~~<br>
> ../include/llvm/ADT/Optional.h:141:3: note: in instantiation of member<br>
> function<br>
> 'llvm::optional_detail::OptionalStorage<clang::clangd::SymbolSlab,<br>
> false>::OptionalStorage' requested here<br>
> Optional(const Optional &O) = default;<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/index/Serialization.cpp:325:10: note:<br>
> in instantiation of function template specialization<br>
> 'llvm::Expected<clang::clangd::IndexFileIn>::Expected<clang::clangd::IndexFileIn<br>
> <br>
> &>' requested here<br>
> return Result;<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/index/Index.h:324:26: note: copy<br>
> constructor of 'SymbolSlab' is implicitly deleted because field 'Arena'<br>
> has a deleted copy constructor<br>
> llvm::BumpPtrAllocator Arena; // Owns Symbol data that the<br>
> Symbols do<br>
> not.<br>
> ^<br>
> ../include/llvm/Support/Allocator.h:157:3: note: copy constructor is<br>
> implicitly deleted because 'BumpPtrAllocatorImpl<llvm::MallocAllocator,<br>
> 4096, 4096>' has a user-declared move constructor<br>
> BumpPtrAllocatorImpl(BumpPtrAllocatorImpl &&Old)<br>
> ^<br>
> 2 errors generated.<br>
> <br>
> Several buildbots fail the same way.<br>
> <br>
> /Mikael<br>
> <br>
> On 09/04/2018 06:16 PM, Sam McCall via cfe-commits wrote:<br>
> > Author: sammccall<br>
> > Date: Tue Sep 4 09:16:50 2018<br>
> > New Revision: 341375<br>
> ><br>
> > URL: <a href="http://llvm.org/viewvc/llvm-project?rev=341375&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=341375&view=rev</a><br>
> > Log:<br>
> > [clangd] Define a compact binary serialization fomat for symbol<br>
> slab/index.<br>
> ><br>
> > Summary:<br>
> > This is intended to replace the current YAML format for general use.<br>
> > It's ~10x more compact than YAML, and ~40% more compact than<br>
> gzipped YAML:<br>
> > llvmidx.riff = 20M, llvmidx.yaml = 272M, llvmidx.yaml.gz = 32M<br>
> > It's also simpler/faster to read and write.<br>
> ><br>
> > The format is a RIFF container (chunks of (type, size, data)) with:<br>
> > - a compressed string table<br>
> > - simple binary encoding of symbols (with varints for compactness)<br>
> > It can be extended to include occurrences, Dex posting lists, etc.<br>
> ><br>
> > There's no rich backwards-compatibility scheme, but a version<br>
> number is included<br>
> > so we can detect incompatible files and do ad-hoc back-compat.<br>
> ><br>
> > Alternatives considered:<br>
> > - compressed YAML or JSON: bulky and slow to load<br>
> > - llvm bitstream: confusing model and libraries are hard to<br>
> use. My attempt<br>
> > produced slightly larger files, and the code was longer and<br>
> slower.<br>
> > - protobuf or similar: would be really nice (esp for<br>
> back-compat) but the<br>
> > dependency is a big hassle<br>
> > - ad-hoc binary format without a container: it seems clear<br>
> we're going<br>
> > to add posting lists and occurrences here, and that they will<br>
> benefit<br>
> > from sharing a string table. The container makes it easy to debug<br>
> > these pieces in isolation, and make them optional.<br>
> ><br>
> > Reviewers: ioeric<br>
> ><br>
> > Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, mgrang,<br>
> arphaman, kadircet, cfe-commits<br>
> ><br>
> > Differential Revision: <a href="https://reviews.llvm.org/D51585" rel="noreferrer" target="_blank">https://reviews.llvm.org/D51585</a><br>
> ><br>
> > Added:<br>
> > clang-tools-extra/trunk/clangd/RIFF.cpp<br>
> > clang-tools-extra/trunk/clangd/RIFF.h<br>
> > clang-tools-extra/trunk/clangd/index/Serialization.cpp<br>
> > clang-tools-extra/trunk/clangd/index/Serialization.h<br>
> > clang-tools-extra/trunk/unittests/clangd/RIFFTests.cpp<br>
> > clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp<br>
> > Modified:<br>
> > clang-tools-extra/trunk/clangd/CMakeLists.txt<br>
> > <br>
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> > clang-tools-extra/trunk/clangd/index/Index.cpp<br>
> > clang-tools-extra/trunk/clangd/index/Index.h<br>
> > clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp<br>
> > clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp<br>
> > clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt<br>
> > <br>
> clang-tools-extra/trunk/unittests/clangd/SymbolCollectorTests.cpp<br>
> ><br>
> > Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> ><br>
> ==============================================================================<br>
> > --- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)<br>
> > +++ clang-tools-extra/trunk/clangd/CMakeLists.txt Tue Sep 4<br>
> 09:16:50 2018<br>
> > @@ -29,6 +29,7 @@ add_clang_library(clangDaemon<br>
> > Protocol.cpp<br>
> > ProtocolHandlers.cpp<br>
> > Quality.cpp<br>
> > + RIFF.cpp<br>
> > SourceCode.cpp<br>
> > Threading.cpp<br>
> > Trace.cpp<br>
> > @@ -41,6 +42,7 @@ add_clang_library(clangDaemon<br>
> > index/Index.cpp<br>
> > index/MemIndex.cpp<br>
> > index/Merge.cpp<br>
> > + index/Serialization.cpp<br>
> > index/SymbolCollector.cpp<br>
> > index/SymbolYAML.cpp<br>
> ><br>
> ><br>
> > Added: clang-tools-extra/trunk/clangd/RIFF.cpp<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.cpp?rev=341375&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.cpp?rev=341375&view=auto</a><br>
> ><br>
> ==============================================================================<br>
> > --- clang-tools-extra/trunk/clangd/RIFF.cpp (added)<br>
> > +++ clang-tools-extra/trunk/clangd/RIFF.cpp Tue Sep 4 09:16:50 2018<br>
> > @@ -0,0 +1,88 @@<br>
> > +//===--- RIFF.cpp - Binary container file format<br>
> --------------------------===//<br>
> > +//<br>
> > +// The LLVM Compiler Infrastructure<br>
> > +//<br>
> > +// This file is distributed under the University of Illinois<br>
> Open Source<br>
> > +// License. See LICENSE.TXT for details.<br>
> > +//<br>
> ><br>
> +//===----------------------------------------------------------------------===//<br>
> > +<br>
> > +#include "RIFF.h"<br>
> > +#include "llvm/Support/Endian.h"<br>
> > +<br>
> > +using namespace llvm;<br>
> > +namespace clang {<br>
> > +namespace clangd {<br>
> > +namespace riff {<br>
> > +<br>
> > +static Error makeError(const char *Msg) {<br>
> > + return createStringError(inconvertibleErrorCode(), Msg);<br>
> > +}<br>
> > +<br>
> > +Expected<Chunk> readChunk(StringRef &Stream) {<br>
> > + if (Stream.size() < 8)<br>
> > + return makeError("incomplete chunk header");<br>
> > + Chunk C;<br>
> > + std::copy(Stream.begin(), Stream.begin() + 4, C.ID.begin());<br>
> > + Stream = Stream.drop_front(4);<br>
> > + uint32_t Len =<br>
> support::endian::read32le(Stream.take_front(4).begin());<br>
> > + Stream = Stream.drop_front(4);<br>
> > + if (Stream.size() < Len)<br>
> > + return makeError("truncated chunk");<br>
> > + C.Data = Stream.take_front(Len);<br>
> > + Stream = Stream.drop_front(Len);<br>
> > + if (Len % 2 & !Stream.empty()) { // Skip padding byte.<br>
> > + if (Stream.front())<br>
> > + return makeError("nonzero padding byte");<br>
> > + Stream = Stream.drop_front();<br>
> > + }<br>
> > + return C;<br>
> > +};<br>
> > +<br>
> > +raw_ostream &operator<<(raw_ostream &OS, const Chunk &C) {<br>
> > + OS.write(C.ID.begin(), C.ID.size());<br>
> > + char Size[4];<br>
> > + llvm::support::endian::write32le(Size, C.Data.size());<br>
> > + OS.write(Size, sizeof(Size));<br>
> > + OS << C.Data;<br>
> > + if (C.Data.size() % 2)<br>
> > + OS.write(0);<br>
> > + return OS;<br>
> > +}<br>
> > +<br>
> > +llvm::Expected<File> readFile(llvm::StringRef Stream) {<br>
> > + auto RIFF = readChunk(Stream);<br>
> > + if (!RIFF)<br>
> > + return RIFF.takeError();<br>
> > + if (RIFF->ID != fourCC("RIFF"))<br>
> > + return makeError("not a RIFF container");<br>
> > + if (RIFF->Data.size() < 4)<br>
> > + return makeError("RIFF chunk too short");<br>
> > + File F;<br>
> > + std::copy(RIFF->Data.begin(), RIFF->Data.begin() + 4,<br>
> F.Type.begin());<br>
> > + for (llvm::StringRef Body = RIFF->Data.drop_front(4);<br>
> !Body.empty();)<br>
> > + if (auto Chunk = readChunk(Body)) {<br>
> > + F.Chunks.push_back(*Chunk);<br>
> > + } else<br>
> > + return Chunk.takeError();<br>
> > + return F;<br>
> > +}<br>
> > +<br>
> > +raw_ostream &operator<<(raw_ostream &OS, const File &F) {<br>
> > + // To avoid copies, we serialize the outer RIFF chunk "by hand".<br>
> > + size_t DataLen = 4; // Predict length of RIFF chunk data.<br>
> > + for (const auto &C : F.Chunks)<br>
> > + DataLen += 4 + 4 + C.Data.size() + (C.Data.size() % 2);<br>
> > + OS << "RIFF";<br>
> > + char Size[4];<br>
> > + llvm::support::endian::write32le(Size, DataLen);<br>
> > + OS.write(Size, sizeof(Size));<br>
> > + OS.write(F.Type.begin(), F.Type.size());<br>
> > + for (const auto &C : F.Chunks)<br>
> > + OS << C;<br>
> > + return OS;<br>
> > +}<br>
> > +<br>
> > +} // namespace riff<br>
> > +} // namespace clangd<br>
> > +} // namespace clang<br>
> ><br>
> > Added: clang-tools-extra/trunk/clangd/RIFF.h<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.h?rev=341375&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.h?rev=341375&view=auto</a><br>
> ><br>
> ==============================================================================<br>
> > --- clang-tools-extra/trunk/clangd/RIFF.h (added)<br>
> > +++ clang-tools-extra/trunk/clangd/RIFF.h Tue Sep 4 09:16:50 2018<br>
> > @@ -0,0 +1,81 @@<br>
> > +//===--- RIFF.h - Binary container file format<br>
> -------------------*- C++-*-===//<br>
> > +//<br>
> > +// The LLVM Compiler Infrastructure<br>
> > +//<br>
> > +// This file is distributed under the University of Illinois<br>
> Open Source<br>
> > +// License. See LICENSE.TXT for details.<br>
> > +//<br>
> ><br>
> +//===----------------------------------------------------------------------===//<br>
> > +//<br>
> > +// Tools for reading and writing data in RIFF containers.<br>
> > +//<br>
> > +// A chunk consists of:<br>
> > +// - ID : char[4]<br>
> > +// - Length : uint32<br>
> > +// - Data : byte[Length]<br>
> > +// - Padding : byte[Length % 2]<br>
> > +// The semantics of a chunk's Data are determined by its ID.<br>
> > +// The format makes it easy to skip over uninteresting or<br>
> unknown chunks.<br>
> > +//<br>
> > +// A RIFF file is a single chunk with ID "RIFF". Its Data is:<br>
> > +// - Type : char[4]<br>
> > +// - Chunks : chunk[]<br>
> > +//<br>
> > +// This means that a RIFF file consists of:<br>
> > +// - "RIFF" : char[4]<br>
> > +// - File length - 8 : uint32<br>
> > +// - File type : char[4]<br>
> > +// - Chunks : chunk[]<br>
> > +//<br>
> ><br>
> +//===----------------------------------------------------------------------===//<br>
> > +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_RIFF_H<br>
> > +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_RIFF_H<br>
> > +#include "llvm/ADT/StringRef.h"<br>
> > +#include "llvm/Support/Error.h"<br>
> > +#include "llvm/Support/ScopedPrinter.h"<br>
> > +#include <array><br>
> > +<br>
> > +namespace clang {<br>
> > +namespace clangd {<br>
> > +namespace riff {<br>
> > +<br>
> > +// A FourCC identifies a chunk in a file, or the type of file<br>
> itself.<br>
> > +using FourCC = std::array<char, 4>;<br>
> > +// Get a FourCC from a string literal, e.g. fourCC("RIFF").<br>
> > +inline constexpr FourCC fourCC(const char (&Literal)[5]) {<br>
> > + return FourCC{{Literal[0], Literal[1], Literal[2], Literal[3]}};<br>
> > +}<br>
> > +// A chunk is a section in a RIFF container.<br>
> > +struct Chunk {<br>
> > + FourCC ID;<br>
> > + llvm::StringRef Data;<br>
> > +};<br>
> > +inline bool operator==(const Chunk &L, const Chunk &R) {<br>
> > + return std::tie(<a href="http://L.ID" rel="noreferrer" target="_blank">L.ID</a> <<a href="http://L.ID" rel="noreferrer" target="_blank">http://L.ID</a>>, L.Data) == std::tie(<a href="http://R.ID" rel="noreferrer" target="_blank">R.ID</a><br>
> <<a href="http://R.ID" rel="noreferrer" target="_blank">http://R.ID</a>>, R.Data);<br>
> > +}<br>
> > +// A File is a RIFF container, which is a typed chunk sequence.<br>
> > +struct File {<br>
> > + FourCC Type;<br>
> > + std::vector<Chunk> Chunks;<br>
> > +};<br>
> > +inline bool operator==(const File &L, const File &R) {<br>
> > + return std::tie(L.Type, L.Chunks) == std::tie(R.Type, R.Chunks);<br>
> > +}<br>
> > +<br>
> > +// Reads a single chunk from the start of Stream.<br>
> > +// Stream is updated to exclude the consumed chunk.<br>
> > +llvm::Expected<Chunk> readChunk(llvm::StringRef &Stream);<br>
> > +<br>
> > +// Serialize a single chunk to OS.<br>
> > +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Chunk &);<br>
> > +<br>
> > +// Parses a RIFF file consisting of a single RIFF chunk.<br>
> > +llvm::Expected<File> readFile(llvm::StringRef Stream);<br>
> > +<br>
> > +// Serialize a RIFF file (i.e. a single RIFF chunk) to OS.<br>
> > +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const File &);<br>
> > +<br>
> > +} // namespace riff<br>
> > +} // namespace clangd<br>
> > +} // namespace clang<br>
> > +#endif<br>
> ><br>
> > Modified:<br>
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> ><br>
> ==============================================================================<br>
> > ---<br>
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> (original)<br>
> > +++<br>
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> Tue Sep 4 09:16:50 2018<br>
> > @@ -7,15 +7,16 @@<br>
> > //<br>
> > <br>
> //===----------------------------------------------------------------------===//<br>
> > //<br>
> > -// GlobalSymbolBuilder is a tool to generate YAML-format symbols<br>
> across the<br>
> > -// whole project. This tools is for **experimental** only. Don't<br>
> use it in<br>
> > -// production code.<br>
> > +// GlobalSymbolBuilder is a tool to extract symbols from a whole<br>
> project.<br>
> > +// This tool is **experimental** only. Don't use it in<br>
> production code.<br>
> > //<br>
> > <br>
> //===----------------------------------------------------------------------===//<br>
> ><br>
> > +#include "RIFF.h"<br>
> > #include "index/CanonicalIncludes.h"<br>
> > #include "index/Index.h"<br>
> > #include "index/Merge.h"<br>
> > +#include "index/Serialization.h"<br>
> > #include "index/SymbolCollector.h"<br>
> > #include "index/SymbolYAML.h"<br>
> > #include "clang/Frontend/CompilerInstance.h"<br>
> > @@ -59,6 +60,14 @@ static llvm::cl::opt<bool> MergeOnTheFly<br>
> > "MapReduce."),<br>
> > llvm::cl::init(true), llvm::cl::Hidden);<br>
> ><br>
> > +enum class Format { YAML, Binary };<br>
> > +static llvm::cl::opt<Format><br>
> > + Format("format", llvm::cl::desc("Format of the index to be<br>
> written"),<br>
> > + llvm::cl::values(<br>
> > + clEnumValN(Format::YAML, "yaml", "human-readable<br>
> YAML format"),<br>
> > + clEnumValN(Format::Binary, "binary", "binary RIFF<br>
> format")),<br>
> > + llvm::cl::init(Format::YAML));<br>
> > +<br>
> > /// Responsible for aggregating symbols from each processed<br>
> file and producing<br>
> > /// the final results. All methods in this class must be<br>
> thread-safe,<br>
> > /// 'consumeSymbols' may be called from multiple threads.<br>
> > @@ -210,8 +219,8 @@ int main(int argc, const char **argv) {<br>
> > llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);<br>
> ><br>
> > const char *Overview = R"(<br>
> > - This is an **experimental** tool to generate YAML-format<br>
> project-wide symbols<br>
> > - for clangd (global code completion). It would be changed and<br>
> deprecated<br>
> > + This is an **experimental** tool to extract symbols from a<br>
> whole project<br>
> > + for clangd (global code completion). It will be changed and<br>
> deprecated<br>
> > eventually. Don't use it in production code!<br>
> ><br>
> > Example usage for building index for the whole project using<br>
> CMake compile<br>
> > @@ -262,7 +271,16 @@ int main(int argc, const char **argv) {<br>
> > }<br>
> > // Reduce phase: combine symbols with the same IDs.<br>
> > auto UniqueSymbols = Consumer->mergeResults();<br>
> > - // Output phase: emit YAML for result symbols.<br>
> > - SymbolsToYAML(UniqueSymbols, llvm::outs());<br>
> > + // Output phase: emit result symbols.<br>
> > + switch (clang::clangd::Format) {<br>
> > + case clang::clangd::Format::YAML:<br>
> > + SymbolsToYAML(UniqueSymbols, llvm::outs());<br>
> > + break;<br>
> > + case clang::clangd::Format::Binary: {<br>
> > + clang::clangd::IndexFileOut Out;<br>
> > + Out.Symbols = &UniqueSymbols;<br>
> > + llvm::outs() << Out;<br>
> > + }<br>
> > + }<br>
> > return 0;<br>
> > }<br>
> ><br>
> > Modified: clang-tools-extra/trunk/clangd/index/Index.cpp<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> ><br>
> ==============================================================================<br>
> > --- clang-tools-extra/trunk/clangd/index/Index.cpp (original)<br>
> > +++ clang-tools-extra/trunk/clangd/index/Index.cpp Tue Sep 4<br>
> 09:16:50 2018<br>
> > @@ -10,6 +10,7 @@<br>
> > #include "Index.h"<br>
> > #include "llvm/ADT/StringExtras.h"<br>
> > #include "llvm/ADT/StringRef.h"<br>
> > +#include "llvm/Support/Error.h"<br>
> > #include "llvm/Support/SHA1.h"<br>
> > #include "llvm/Support/raw_ostream.h"<br>
> ><br>
> > @@ -28,21 +29,20 @@ SymbolID::SymbolID(StringRef USR)<br>
> > : HashValue(SHA1::hash(arrayRefFromStringRef(USR))) {}<br>
> ><br>
> > raw_ostream &operator<<(raw_ostream &OS, const SymbolID &ID) {<br>
> > - OS << toHex(toStringRef(ID.HashValue));<br>
> > - return OS;<br>
> > + return OS << toHex(ID.raw());<br>
> > }<br>
> ><br>
> > -std::string SymbolID::str() const {<br>
> > - std::string ID;<br>
> > - llvm::raw_string_ostream OS(ID);<br>
> > - OS << *this;<br>
> > - return OS.str();<br>
> > +SymbolID SymbolID::fromRaw(llvm::StringRef Raw) {<br>
> > + SymbolID ID;<br>
> > + assert(Raw.size() == RawSize);<br>
> > + memcpy(ID.HashValue.data(), Raw.data(), RawSize);<br>
> > + return ID;<br>
> > }<br>
> ><br>
> > +std::string SymbolID::str() const { return toHex(raw()); }<br>
> > +<br>
> > void operator>>(StringRef Str, SymbolID &ID) {<br>
> > - std::string HexString = fromHex(Str);<br>
> > - assert(HexString.size() == ID.HashValue.size());<br>
> > - std::copy(HexString.begin(), HexString.end(),<br>
> ID.HashValue.begin());<br>
> > + ID = SymbolID::fromRaw(fromHex(Str));<br>
> > }<br>
> ><br>
> > raw_ostream &operator<<(raw_ostream &OS, SymbolOrigin O) {<br>
> > @@ -78,34 +78,18 @@ SymbolSlab::const_iterator SymbolSlab::f<br>
> > }<br>
> ><br>
> > // Copy the underlying data of the symbol into the owned arena.<br>
> > -static void own(Symbol &S, llvm::UniqueStringSaver &Strings,<br>
> > - BumpPtrAllocator &Arena) {<br>
> > - // Intern replaces V with a reference to the same string owned<br>
> by the arena.<br>
> > - auto Intern = [&](StringRef &V) { V = Strings.save(V); };<br>
> > -<br>
> > - // We need to copy every StringRef field onto the arena.<br>
> > - Intern(S.Name);<br>
> > - Intern(S.Scope);<br>
> > - Intern(S.CanonicalDeclaration.FileURI);<br>
> > - Intern(S.Definition.FileURI);<br>
> > -<br>
> > - Intern(S.Signature);<br>
> > - Intern(S.CompletionSnippetSuffix);<br>
> > -<br>
> > - Intern(S.Documentation);<br>
> > - Intern(S.ReturnType);<br>
> > - for (auto &I : S.IncludeHeaders)<br>
> > - Intern(I.IncludeHeader);<br>
> > +static void own(Symbol &S, llvm::UniqueStringSaver &Strings) {<br>
> > + visitStrings(S, [&](StringRef &V) { V = Strings.save(V); });<br>
> > }<br>
> ><br>
> > void SymbolSlab::Builder::insert(const Symbol &S) {<br>
> > auto R = SymbolIndex.try_emplace(<a href="http://S.ID" rel="noreferrer" target="_blank">S.ID</a> <<a href="http://S.ID" rel="noreferrer" target="_blank">http://S.ID</a>>,<br>
> Symbols.size());<br>
> > if (R.second) {<br>
> > Symbols.push_back(S);<br>
> > - own(Symbols.back(), UniqueStrings, Arena);<br>
> > + own(Symbols.back(), UniqueStrings);<br>
> > } else {<br>
> > auto &Copy = Symbols[R.first->second] = S;<br>
> > - own(Copy, UniqueStrings, Arena);<br>
> > + own(Copy, UniqueStrings);<br>
> > }<br>
> > }<br>
> ><br>
> > @@ -118,7 +102,7 @@ SymbolSlab SymbolSlab::Builder::build()<br>
> > BumpPtrAllocator NewArena;<br>
> > llvm::UniqueStringSaver Strings(NewArena);<br>
> > for (auto &S : Symbols)<br>
> > - own(S, Strings, NewArena);<br>
> > + own(S, Strings);<br>
> > return SymbolSlab(std::move(NewArena), std::move(Symbols));<br>
> > }<br>
> ><br>
> ><br>
> > Modified: clang-tools-extra/trunk/clangd/index/Index.h<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> ><br>
> ==============================================================================<br>
> > --- clang-tools-extra/trunk/clangd/index/Index.h (original)<br>
> > +++ clang-tools-extra/trunk/clangd/index/Index.h Tue Sep 4<br>
> 09:16:50 2018<br>
> > @@ -84,26 +84,28 @@ public:<br>
> > return HashValue < Sym.HashValue;<br>
> > }<br>
> ><br>
> > + constexpr static size_t RawSize = 20;<br>
> > + llvm::StringRef raw() const {<br>
> > + return StringRef(reinterpret_cast<const char<br>
> *>(HashValue.data()), RawSize);<br>
> > + }<br>
> > + static SymbolID fromRaw(llvm::StringRef);<br>
> > // Returns a 40-bytes hex encoded string.<br>
> > std::string str() const;<br>
> ><br>
> > private:<br>
> > - static constexpr unsigned HashByteLength = 20;<br>
> > -<br>
> > - friend llvm::hash_code hash_value(const SymbolID &ID) {<br>
> > - // We already have a good hash, just return the first bytes.<br>
> > - static_assert(sizeof(size_t) <= HashByteLength, "size_t<br>
> longer than SHA1!");<br>
> > - size_t Result;<br>
> > - memcpy(&Result, ID.HashValue.data(), sizeof(size_t));<br>
> > - return llvm::hash_code(Result);<br>
> > - }<br>
> > - friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,<br>
> > - const SymbolID &ID);<br>
> > friend void operator>>(llvm::StringRef Str, SymbolID &ID);<br>
> ><br>
> > - std::array<uint8_t, HashByteLength> HashValue;<br>
> > + std::array<uint8_t, RawSize> HashValue;<br>
> > };<br>
> ><br>
> > +inline llvm::hash_code hash_value(const SymbolID &ID) {<br>
> > + // We already have a good hash, just return the first bytes.<br>
> > + assert(sizeof(size_t) <= SymbolID::RawSize && "size_t longer<br>
> than SHA1!");<br>
> > + size_t Result;<br>
> > + memcpy(&Result, ID.raw().data(), sizeof(size_t));<br>
> > + return llvm::hash_code(Result);<br>
> > +}<br>
> > +<br>
> > // Write SymbolID into the given stream. SymbolID is encoded as<br>
> a 40-bytes<br>
> > // hex string.<br>
> > llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const<br>
> SymbolID &ID);<br>
> > @@ -246,6 +248,21 @@ struct Symbol {<br>
> > };<br>
> > llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const<br>
> Symbol &S);<br>
> ><br>
> > +// Invokes Callback with each StringRef& contained in the Symbol.<br>
> > +// Useful for deduplicating backing strings.<br>
> > +template <typename Callback> void visitStrings(Symbol &S, const<br>
> Callback &CB) {<br>
> > + CB(S.Name);<br>
> > + CB(S.Scope);<br>
> > + CB(S.CanonicalDeclaration.FileURI);<br>
> > + CB(S.Definition.FileURI);<br>
> > + CB(S.Signature);<br>
> > + CB(S.CompletionSnippetSuffix);<br>
> > + CB(S.Documentation);<br>
> > + CB(S.ReturnType);<br>
> > + for (auto &Include : S.IncludeHeaders)<br>
> > + CB(Include.IncludeHeader);<br>
> > +}<br>
> > +<br>
> > // Computes query-independent quality score for a Symbol.<br>
> > // This currently falls in the range [1, ln(#indexed documents)].<br>
> > // FIXME: this should probably be split into symbol -> signals<br>
> ><br>
> > Added: clang-tools-extra/trunk/clangd/index/Serialization.cpp<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=341375&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=341375&view=auto</a><br>
> ><br>
> ==============================================================================<br>
> > --- clang-tools-extra/trunk/clangd/index/Serialization.cpp (added)<br>
> > +++ clang-tools-extra/trunk/clangd/index/Serialization.cpp Tue<br>
> Sep 4 09:16:50 2018<br>
> > @@ -0,0 +1,366 @@<br>
> > +//===-- Serialization.cpp - Binary serialization of index data<br>
> ------------===//<br>
> > +//<br>
> > +// The LLVM Compiler Infrastructure<br>
> > +//<br>
> > +// This file is distributed under the University of Illinois<br>
> Open Source<br>
> > +// License. See LICENSE.TXT for details.<br>
> > +//<br>
> ><br>
> +//===----------------------------------------------------------------------===//<br>
> > +#include "Serialization.h"<br>
> > +#include "../RIFF.h"<br>
> > +#include "llvm/Support/Compression.h"<br>
> > +#include "llvm/Support/Endian.h"<br>
> > +#include "llvm/Support/Error.h"<br>
> > +<br>
> > +using namespace llvm;<br>
> > +namespace clang {<br>
> > +namespace clangd {<br>
> > +namespace {<br>
> > +Error makeError(const Twine &Msg) {<br>
> > + return make_error<StringError>(Msg, inconvertibleErrorCode());<br>
> > +}<br>
> > +<br>
> > +// IO PRIMITIVES<br>
> > +// We use little-endian 32 bit ints, sometimes with<br>
> variable-length encoding.<br>
> > +<br>
> > +StringRef consume(StringRef &Data, int N) {<br>
> > + StringRef Ret = Data.take_front(N);<br>
> > + Data = Data.drop_front(N);<br>
> > + return Ret;<br>
> > +}<br>
> > +<br>
> > +uint8_t consume8(StringRef &Data) {<br>
> > + uint8_t Ret = Data.front();<br>
> > + Data = Data.drop_front();<br>
> > + return Ret;<br>
> > +}<br>
> > +<br>
> > +uint32_t consume32(StringRef &Data) {<br>
> > + auto Ret = support::endian::read32le(Data.bytes_begin());<br>
> > + Data = Data.drop_front(4);<br>
> > + return Ret;<br>
> > +}<br>
> > +<br>
> > +void write32(uint32_t I, raw_ostream &OS) {<br>
> > + char buf[4];<br>
> > + support::endian::write32le(buf, I);<br>
> > + OS.write(buf, sizeof(buf));<br>
> > +}<br>
> > +<br>
> > +// Variable-length int encoding (varint) uses the bottom 7 bits<br>
> of each byte<br>
> > +// to encode the number, and the top bit to indicate whether<br>
> more bytes follow.<br>
> > +// e.g. 9a 2f means [0x1a and keep reading, 0x2f and stop].<br>
> > +// This represents 0x1a | 0x2f<<7 = 6042.<br>
> > +// A 32-bit integer takes 1-5 bytes to encode; small numbers are<br>
> more compact.<br>
> > +void writeVar(uint32_t I, raw_ostream &OS) {<br>
> > + constexpr static uint8_t More = 1 << 7;<br>
> > + if (LLVM_LIKELY(I < 1 << 7)) {<br>
> > + OS.write(I);<br>
> > + return;<br>
> > + }<br>
> > + for (;;) {<br>
> > + OS.write(I | More);<br>
> > + I >>= 7;<br>
> > + if (I < 1 << 7) {<br>
> > + OS.write(I);<br>
> > + return;<br>
> > + }<br>
> > + }<br>
> > +}<br>
> > +<br>
> > +uint32_t consumeVar(StringRef &Data) {<br>
> > + constexpr static uint8_t More = 1 << 7;<br>
> > + uint8_t B = consume8(Data);<br>
> > + if (LLVM_LIKELY(!(B & More)))<br>
> > + return B;<br>
> > + uint32_t Val = B & ~More;<br>
> > + for (int Shift = 7; B & More && Shift < 32; Shift += 7) {<br>
> > + B = consume8(Data);<br>
> > + Val |= (B & ~More) << Shift;<br>
> > + }<br>
> > + return Val;<br>
> > +}<br>
> > +<br>
> > +// STRING TABLE ENCODING<br>
> > +// Index data has many string fields, and many strings are<br>
> identical.<br>
> > +// We store each string once, and refer to them by index.<br>
> > +//<br>
> > +// The string table's format is:<br>
> > +// - UncompressedSize : uint32<br>
> > +// - CompressedData : byte[CompressedSize]<br>
> > +//<br>
> > +// CompressedData is a zlib-compressed byte[UncompressedSize].<br>
> > +// It contains a sequence of null-terminated strings, e.g.<br>
> "foo\0bar\0".<br>
> > +// These are sorted to improve compression.<br>
> > +<br>
> > +// Maps each string to a canonical representation.<br>
> > +// Strings remain owned externally (e.g. by SymbolSlab).<br>
> > +class StringTableOut {<br>
> > + DenseSet<StringRef> Unique;<br>
> > + std::vector<StringRef> Sorted;<br>
> > + // Since strings are interned, look up can be by pointer.<br>
> > + DenseMap<std::pair<const char *, size_t>, unsigned> Index;<br>
> > +<br>
> > +public:<br>
> > + // Add a string to the table. Overwrites S if an identical<br>
> string exists.<br>
> > + void intern(StringRef &S) { S = *Unique.insert(S).first; };<br>
> > + // Finalize the table and write it to OS. No more strings may<br>
> be added.<br>
> > + void finalize(raw_ostream &OS) {<br>
> > + Sorted = {Unique.begin(), Unique.end()};<br>
> > + std::sort(Sorted.begin(), Sorted.end());<br>
> > + for (unsigned I = 0; I < Sorted.size(); ++I)<br>
> > + Index.try_emplace({Sorted[I].data(), Sorted[I].size()}, I);<br>
> > +<br>
> > + std::string RawTable;<br>
> > + for (StringRef S : Sorted) {<br>
> > + RawTable.append(S);<br>
> > + RawTable.push_back(0);<br>
> > + }<br>
> > + SmallString<1> Compressed;<br>
> > + cantFail(zlib::compress(RawTable, Compressed));<br>
> > + write32(RawTable.size(), OS);<br>
> > + OS << Compressed;<br>
> > + }<br>
> > + // Get the ID of an string, which must be interned. Table must<br>
> be finalized.<br>
> > + unsigned index(StringRef S) const {<br>
> > + assert(!Sorted.empty() && "table not finalized");<br>
> > + assert(Index.count({S.data(), S.size()}) && "string not<br>
> interned");<br>
> > + return Index.find({S.data(), S.size()})->second;<br>
> > + }<br>
> > +};<br>
> > +<br>
> > +struct StringTableIn {<br>
> > + BumpPtrAllocator Arena;<br>
> > + std::vector<StringRef> Strings;<br>
> > +};<br>
> > +<br>
> > +Expected<StringTableIn> readStringTable(StringRef Data) {<br>
> > + if (Data.size() < 4)<br>
> > + return makeError("Bad string table: not enough metadata");<br>
> > + size_t UncompressedSize = consume32(Data);<br>
> > + SmallString<1> Uncompressed;<br>
> > + if (Error E = llvm::zlib::uncompress(Data, Uncompressed,<br>
> UncompressedSize))<br>
> > + return std::move(E);<br>
> > +<br>
> > + StringTableIn Table;<br>
> > + StringSaver Saver(Table.Arena);<br>
> > + for (StringRef Rest = Uncompressed; !Rest.empty();) {<br>
> > + auto Len = Rest.find(0);<br>
> > + if (Len == StringRef::npos)<br>
> > + return makeError("Bad string table: not null terminated");<br>
> > + Table.Strings.push_back(Saver.save(consume(Rest, Len)));<br>
> > + Rest = Rest.drop_front();<br>
> > + }<br>
> > + return Table;<br>
> > +}<br>
> > +<br>
> > +// SYMBOL ENCODING<br>
> > +// Each field of clangd::Symbol is encoded in turn (see<br>
> implementation).<br>
> > +// - StringRef fields encode as varint (index into the string<br>
> table)<br>
> > +// - enums encode as the underlying type<br>
> > +// - most numbers encode as varint<br>
> > +<br>
> > +// It's useful to the implementation to assume symbols have a<br>
> bounded size.<br>
> > +constexpr size_t SymbolSizeBound = 512;<br>
> > +// To ensure the bounded size, restrict the number of include<br>
> headers stored.<br>
> > +constexpr unsigned MaxIncludes = 50;<br>
> > +<br>
> > +void writeSymbol(const Symbol &Sym, const StringTableOut &Strings,<br>
> > + raw_ostream &OS) {<br>
> > + auto StartOffset = OS.tell();<br>
> > + OS << Sym.ID.raw(); // TODO: once we start writing xrefs and<br>
> posting lists,<br>
> > + // symbol IDs should probably be in a<br>
> string table.<br>
> > + OS.write(static_cast<uint8_t>(Sym.SymInfo.Kind));<br>
> > + OS.write(static_cast<uint8_t>(Sym.SymInfo.Lang));<br>
> > + writeVar(Strings.index(Sym.Name), OS);<br>
> > + writeVar(Strings.index(Sym.Scope), OS);<br>
> > + for (const auto &Loc : {Sym.Definition,<br>
> Sym.CanonicalDeclaration}) {<br>
> > + writeVar(Strings.index(Loc.FileURI), OS);<br>
> > + for (const auto &Endpoint : {Loc.Start, Loc.End}) {<br>
> > + writeVar(Endpoint.Line, OS);<br>
> > + writeVar(Endpoint.Column, OS);<br>
> > + }<br>
> > + }<br>
> > + writeVar(Sym.References, OS);<br>
> > + OS.write(Sym.IsIndexedForCodeCompletion);<br>
> > + OS.write(static_cast<uint8_t>(Sym.Origin));<br>
> > + writeVar(Strings.index(Sym.Signature), OS);<br>
> > + writeVar(Strings.index(Sym.CompletionSnippetSuffix), OS);<br>
> > + writeVar(Strings.index(Sym.Documentation), OS);<br>
> > + writeVar(Strings.index(Sym.ReturnType), OS);<br>
> > +<br>
> > + auto WriteInclude = [&](const<br>
> Symbol::IncludeHeaderWithReferences &Include) {<br>
> > + writeVar(Strings.index(Include.IncludeHeader), OS);<br>
> > + writeVar(Include.References, OS);<br>
> > + };<br>
> > + // There are almost certainly few includes, so we can just<br>
> write them.<br>
> > + if (LLVM_LIKELY(Sym.IncludeHeaders.size() <= MaxIncludes)) {<br>
> > + writeVar(Sym.IncludeHeaders.size(), OS);<br>
> > + for (const auto &Include : Sym.IncludeHeaders)<br>
> > + WriteInclude(Include);<br>
> > + } else {<br>
> > + // If there are too many, make sure we truncate the least<br>
> important.<br>
> > + using Pointer = const Symbol::IncludeHeaderWithReferences *;<br>
> > + std::vector<Pointer> Pointers;<br>
> > + for (const auto &Include : Sym.IncludeHeaders)<br>
> > + Pointers.push_back(&Include);<br>
> > + std::sort(Pointers.begin(), Pointers.end(), [](Pointer L,<br>
> Pointer R) {<br>
> > + return L->References > R->References;<br>
> > + });<br>
> > + Pointers.resize(MaxIncludes);<br>
> > +<br>
> > + writeVar(MaxIncludes, OS);<br>
> > + for (Pointer P : Pointers)<br>
> > + WriteInclude(*P);<br>
> > + }<br>
> > +<br>
> > + assert(OS.tell() - StartOffset < SymbolSizeBound && "Symbol<br>
> length unsafe!");<br>
> > + (void)StartOffset; // Unused in NDEBUG;<br>
> > +}<br>
> > +<br>
> > +Expected<Symbol> readSymbol(StringRef &Data, const StringTableIn<br>
> &Strings) {<br>
> > + // Usually we can skip bounds checks because the buffer is huge.<br>
> > + // Near the end of the buffer, this would be unsafe. In this<br>
> rare case, copy<br>
> > + // the data into a bigger buffer so we can again skip the checks.<br>
> > + if (LLVM_UNLIKELY(Data.size() < SymbolSizeBound)) {<br>
> > + std::string Buf(Data);<br>
> > + Buf.resize(SymbolSizeBound);<br>
> > + StringRef ExtendedData = Buf;<br>
> > + auto Ret = readSymbol(ExtendedData, Strings);<br>
> > + unsigned BytesRead = Buf.size() - ExtendedData.size();<br>
> > + if (BytesRead > Data.size())<br>
> > + return makeError("read past end of data");<br>
> > + Data = Data.drop_front(BytesRead);<br>
> > + return Ret;<br>
> > + }<br>
> > +<br>
> > +#define READ_STRING(Field) <br>
> \<br>
> > + do { <br>
> \<br>
> > + auto StringIndex = consumeVar(Data); <br>
> \<br>
> > + if (LLVM_UNLIKELY(StringIndex >= Strings.Strings.size())) <br>
> \<br>
> > + return makeError("Bad string index"); <br>
> \<br>
> > + Field = Strings.Strings[StringIndex]; <br>
> \<br>
> > + } while (0)<br>
> > +<br>
> > + Symbol Sym;<br>
> > + Sym.ID = SymbolID::fromRaw(consume(Data, 20));<br>
> > + Sym.SymInfo.Kind = static_cast<index::SymbolKind>(consume8(Data));<br>
> > + Sym.SymInfo.Lang =<br>
> static_cast<index::SymbolLanguage>(consume8(Data));<br>
> > + READ_STRING(Sym.Name);<br>
> > + READ_STRING(Sym.Scope);<br>
> > + for (SymbolLocation *Loc : {&Sym.Definition,<br>
> &Sym.CanonicalDeclaration}) {<br>
> > + READ_STRING(Loc->FileURI);<br>
> > + for (auto &Endpoint : {&Loc->Start, &Loc->End}) {<br>
> > + Endpoint->Line = consumeVar(Data);<br>
> > + Endpoint->Column = consumeVar(Data);<br>
> > + }<br>
> > + }<br>
> > + Sym.References = consumeVar(Data);<br>
> > + Sym.IsIndexedForCodeCompletion = consume8(Data);<br>
> > + Sym.Origin = static_cast<SymbolOrigin>(consume8(Data));<br>
> > + READ_STRING(Sym.Signature);<br>
> > + READ_STRING(Sym.CompletionSnippetSuffix);<br>
> > + READ_STRING(Sym.Documentation);<br>
> > + READ_STRING(Sym.ReturnType);<br>
> > + unsigned IncludeHeaderN = consumeVar(Data);<br>
> > + if (IncludeHeaderN > MaxIncludes)<br>
> > + return makeError("too many IncludeHeaders");<br>
> > + Sym.IncludeHeaders.resize(IncludeHeaderN);<br>
> > + for (auto &I : Sym.IncludeHeaders) {<br>
> > + READ_STRING(I.IncludeHeader);<br>
> > + I.References = consumeVar(Data);<br>
> > + }<br>
> > +<br>
> > +#undef READ_STRING<br>
> > + return Sym;<br>
> > +}<br>
> > +<br>
> > +} // namespace<br>
> > +<br>
> > +// FILE ENCODING<br>
> > +// A file is a RIFF chunk with type 'CdIx'.<br>
> > +// It contains the sections:<br>
> > +// - meta: version number<br>
> > +// - stri: string table<br>
> > +// - symb: symbols<br>
> > +<br>
> > +// The current versioning scheme is simple - non-current<br>
> versions are rejected.<br>
> > +// This allows arbitrary format changes, which invalidate stored<br>
> data.<br>
> > +// Later we may want to support some backward compatibility.<br>
> > +constexpr static uint32_t Version = 1;<br>
> > +<br>
> > +Expected<IndexFileIn> readIndexFile(StringRef Data) {<br>
> > + auto RIFF = riff::readFile(Data);<br>
> > + if (!RIFF)<br>
> > + return RIFF.takeError();<br>
> > + if (RIFF->Type != riff::fourCC("CdIx"))<br>
> > + return makeError("wrong RIFF type");<br>
> > + StringMap<StringRef> Chunks;<br>
> > + for (const auto &Chunk : RIFF->Chunks)<br>
> > + Chunks.try_emplace(StringRef(Chunk.ID.data(),<br>
> Chunk.ID.size()), Chunk.Data);<br>
> > +<br>
> > + for (StringRef RequiredChunk : {"meta", "stri"})<br>
> > + if (!Chunks.count(RequiredChunk))<br>
> > + return makeError("missing required chunk " + RequiredChunk);<br>
> > +<br>
> > + StringRef Meta = Chunks.lookup("meta");<br>
> > + if (Meta.size() < 4 || consume32(Meta) != Version)<br>
> > + return makeError("wrong version");<br>
> > +<br>
> > + auto Strings = readStringTable(Chunks.lookup("stri"));<br>
> > + if (!Strings)<br>
> > + return Strings.takeError();<br>
> > +<br>
> > + IndexFileIn Result;<br>
> > + if (Chunks.count("symb")) {<br>
> > + StringRef SymbolData = Chunks.lookup("symb");<br>
> > + SymbolSlab::Builder Symbols;<br>
> > + while (!SymbolData.empty())<br>
> > + if (auto Sym = readSymbol(SymbolData, *Strings))<br>
> > + Symbols.insert(*Sym);<br>
> > + else<br>
> > + return Sym.takeError();<br>
> > + Result.Symbols = std::move(Symbols).build();<br>
> > + }<br>
> > + return Result;<br>
> > +}<br>
> > +<br>
> > +raw_ostream &operator<<(raw_ostream &OS, const IndexFileOut &Data) {<br>
> > + assert(Data.Symbols && "An index file without symbols makes no<br>
> sense!");<br>
> > + riff::File RIFF;<br>
> > + RIFF.Type = riff::fourCC("CdIx");<br>
> > +<br>
> > + SmallString<4> Meta;<br>
> > + {<br>
> > + raw_svector_ostream MetaOS(Meta);<br>
> > + write32(Version, MetaOS);<br>
> > + }<br>
> > + RIFF.Chunks.push_back({riff::fourCC("meta"), Meta});<br>
> > +<br>
> > + StringTableOut Strings;<br>
> > + std::vector<Symbol> Symbols;<br>
> > + for (const auto &Sym : *Data.Symbols) {<br>
> > + Symbols.emplace_back(Sym);<br>
> > + visitStrings(Symbols.back(), [&](StringRef &S) {<br>
> Strings.intern(S); });<br>
> > + }<br>
> > +<br>
> > + std::string StringSection;<br>
> > + {<br>
> > + raw_string_ostream StringOS(StringSection);<br>
> > + Strings.finalize(StringOS);<br>
> > + }<br>
> > + RIFF.Chunks.push_back({riff::fourCC("stri"), StringSection});<br>
> > +<br>
> > + std::string SymbolSection;<br>
> > + {<br>
> > + raw_string_ostream SymbolOS(SymbolSection);<br>
> > + for (const auto &Sym : Symbols)<br>
> > + writeSymbol(Sym, Strings, SymbolOS);<br>
> > + }<br>
> > + RIFF.Chunks.push_back({riff::fourCC("symb"), SymbolSection});<br>
> > +<br>
> > + return OS << RIFF;<br>
> > +}<br>
> > +<br>
> > +} // namespace clangd<br>
> > +} // namespace clang<br>
> ><br>
> > Added: clang-tools-extra/trunk/clangd/index/Serialization.h<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.h?rev=341375&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.h?rev=341375&view=auto</a><br>
> ><br>
> ==============================================================================<br>
> > --- clang-tools-extra/trunk/clangd/index/Serialization.h (added)<br>
> > +++ clang-tools-extra/trunk/clangd/index/Serialization.h Tue Sep <br>
> 4 09:16:50 2018<br>
> > @@ -0,0 +1,48 @@<br>
> > +//===--- Serialization.h - Binary serialization of index data<br>
> ----*- C++-*-===//<br>
> > +//<br>
> > +// The LLVM Compiler Infrastructure<br>
> > +//<br>
> > +// This file is distributed under the University of Illinois<br>
> Open Source<br>
> > +// License. See LICENSE.TXT for details.<br>
> > +//<br>
> ><br>
> +//===----------------------------------------------------------------------===//<br>
> > +//<br>
> > +// This file provides a compact binary serialization of indexed<br>
> symbols.<br>
> > +//<br>
> > +// It writes two sections:<br>
> > +// - a string table (which is compressed)<br>
> > +// - lists of encoded symbols<br>
> > +//<br>
> > +// The format has a simple versioning scheme: the version is<br>
> embedded in the<br>
> > +// data and non-current versions are rejected when reading.<br>
> > +//<br>
> ><br>
> +//===----------------------------------------------------------------------===//<br>
> > +<br>
> > +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_RIFF_H<br>
> > +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_RIFF_H<br>
> > +#include "Index.h"<br>
> > +#include "llvm/Support/Error.h"<br>
> > +<br>
> > +namespace clang {<br>
> > +namespace clangd {<br>
> > +<br>
> > +// Specifies the contents of an index file to be written.<br>
> > +struct IndexFileOut {<br>
> > + const SymbolSlab *Symbols;<br>
> > + // TODO: Support serializing symbol occurrences.<br>
> > + // TODO: Support serializing Dex posting lists.<br>
> > +};<br>
> > +// Serializes an index file. (This is a RIFF container chunk).<br>
> > +llvm::raw_ostream &operator<<(llvm::raw_ostream &, const<br>
> IndexFileOut &);<br>
> > +<br>
> > +// Holds the contents of an index file that was read.<br>
> > +struct IndexFileIn {<br>
> > + llvm::Optional<SymbolSlab> Symbols;<br>
> > +};<br>
> > +// Parse an index file. The input must be a RIFF container chunk.<br>
> > +llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);<br>
> > +<br>
> > +} // namespace clangd<br>
> > +} // namespace clang<br>
> > +<br>
> > +#endif<br>
> ><br>
> > Modified: clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp<br>
> > URL:<br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> ><br>
> ==============================================================================<br>
> > --- clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp (original)<br>
> > +++ clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp Tue Sep <br>
> 4 09:16:50 2018<br>
> > @@ -9,6 +9,7 @@<br>
> ><br>
> > #include "SymbolYAML.h"<br>
> > #include "Index.h"<br>
> > +#include "Serialization.h"<br>
> > #include "dex/DexIndex.h"<br>
> > #include "llvm/ADT/Optional.h"<br>
> > #include "llvm/ADT/SmallVector.h"<br>
> > @@ -189,10 +190,20 @@ std::unique_ptr<SymbolIndex> loadIndex(l<br>
> > llvm::errs() << "Can't open " << SymbolFile << "\n";<br>
> > return nullptr;<br>
> > }<br>
> > - auto Slab = symbolsFromYAML(Buffer.get()->getBuffer());<br>
> > + StringRef Data = Buffer->get()->getBuffer();<br>
> ><br>
> > - return UseDex ? dex::DexIndex::build(std::move(Slab))<br>
> > - : MemIndex::build(std::move(Slab), RefSlab());<br>
> > + llvm::Optional<SymbolSlab> Slab;<br>
> > + if (Data.startswith("RIFF")) { // Magic for binary index file.<br>
> > + if (auto RIFF = readIndexFile(Data))<br>
> > + Slab = std::move(RIFF->Symbols);<br>
> > + else<br>
> > + llvm::errs() << "Bad RIFF: " <<<br>
> </blockquote></div>