<div dir="ltr"><div dir="ltr"><div dir="ltr">Ah, thanks! Fixed in r341467.<div><br></div><div>I'd guess this was a GCC bug, because GCC >=6 accepts it: <a href="https://godbolt.org/z/Yb5gBT">https://godbolt.org/z/Yb5gBT</a><br></div></div></div></div><br><div class="gmail_quote"><div dir="ltr">On Wed, Sep 5, 2018 at 2:21 PM Mikael Holmén <<a href="mailto:mikael.holmen@ericsson.com">mikael.holmen@ericsson.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"><br>
<br>
On 09/05/2018 12:41 PM, Sam McCall wrote:<br>
> Thanks. Unclear to me whether it's the enum class or the anonymous <br>
> namespace that's triggering this (I believe) compiler bug, but r341459 <br>
> may help...<br>
<br>
Still doesn't work.<br>
<br>
In file included from <br>
../tools/clang/include/clang/Frontend/CommandLineSourceLoc.h:19:0,<br>
from <br>
../tools/clang/include/clang/Frontend/FrontendOptions.h:13,<br>
from <br>
../tools/clang/include/clang/Frontend/CompilerInvocation.h:19,<br>
from <br>
../tools/clang/include/clang/Frontend/CompilerInstance.h:16,<br>
from <br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:22:<br>
../include/llvm/Support/CommandLine.h:606:29: error: expected <br>
primary-expression before '{' token<br>
llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:67:16: <br>
note: in expansion of macro 'clEnumValN'<br>
clEnumValN(Format::YAML, "yaml", "human-readable YAML <br>
format"),<br>
^<br>
../include/llvm/Support/CommandLine.h:606:29: error: expected <br>
primary-expression before '{' token<br>
llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:68:16: <br>
note: in expansion of macro 'clEnumValN'<br>
clEnumValN(Format::Binary, "binary", "binary RIFF <br>
format")),<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:69:27: <br>
error: 'Format' is not a class, namespace, or enumeration<br>
llvm::cl::init(Format::YAML));<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp: <br>
In function 'int main(int, const char**)':<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:276:23: <br>
error: 'clang::clangd::Format' is not a class, namespace, or enumeration<br>
case clang::clangd::Format::YAML:<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:279:23: <br>
error: 'clang::clangd::Format' is not a class, namespace, or enumeration<br>
case clang::clangd::Format::Binary: {<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10: <br>
warning: enumeration value 'YAML' not handled in switch [-Wswitch]<br>
switch (clang::clangd::Format) {<br>
^<br>
../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10: <br>
warning: enumeration value 'Binary' not handled in switch [-Wswitch]<br>
<br>
Changing the name of the enum from Format to Formats (so the name isn't <br>
the same as the variable) makes it compile.<br>
<br>
I.e:<br>
<br>
@@ -58,17 +58,17 @@ static llvm::cl::opt<bool> MergeOnTheFly(<br>
"usage and an almost instant reduce stage. Optimal for running <br>
as a "<br>
"standalone tool, but cannot be used with multi-process <br>
executors like "<br>
"MapReduce."),<br>
llvm::cl::init(true), llvm::cl::Hidden);<br>
<br>
-enum Format { YAML, Binary };<br>
-static llvm::cl::opt<Format><br>
+enum Formats { YAML, Binary };<br>
+static llvm::cl::opt<Formats><br>
Format("format", llvm::cl::desc("Format of the index to be written"),<br>
llvm::cl::values(<br>
- clEnumValN(Format::YAML, "yaml", "human-readable YAML <br>
format"),<br>
- clEnumValN(Format::Binary, "binary", "binary RIFF format")),<br>
- llvm::cl::init(Format::YAML));<br>
+ clEnumValN(Formats::YAML, "yaml", "human-readable YAML <br>
format"),<br>
+ clEnumValN(Formats::Binary, "binary", "binary RIFF <br>
format")),<br>
+ llvm::cl::init(Formats::YAML));<br>
<br>
/// Responsible for aggregating symbols from each processed file and <br>
producing<br>
/// the final results. All methods in this class must be thread-safe,<br>
/// 'consumeSymbols' may be called from multiple threads.<br>
class SymbolsConsumer {<br>
@@ -271,14 +271,14 @@ int main(int argc, const char **argv) {<br>
}<br>
// Reduce phase: combine symbols with the same IDs.<br>
auto UniqueSymbols = Consumer->mergeResults();<br>
// Output phase: emit result symbols.<br>
switch (clang::clangd::Format) {<br>
- case clang::clangd::Format::YAML:<br>
+ case clang::clangd::Formats::YAML:<br>
SymbolsToYAML(UniqueSymbols, llvm::outs());<br>
break;<br>
- case clang::clangd::Format::Binary: {<br>
+ case clang::clangd::Formats::Binary: {<br>
clang::clangd::IndexFileOut Out;<br>
Out.Symbols = &UniqueSymbols;<br>
llvm::outs() << Out;<br>
}<br>
}<br>
<br>
seems to compile with gcc 5.4.0.<br>
<br>
I've no idea if this is a gcc bug or if it's a bug in clang to not also <br>
complain about it.<br>
<br>
/Mikael<br>
<br>
> <br>
> On Wed, Sep 5, 2018 at 11:05 AM Mikael Holmén <br>
> <<a href="mailto:mikael.holmen@ericsson.com" target="_blank">mikael.holmen@ericsson.com</a> <mailto:<a href="mailto:mikael.holmen@ericsson.com" target="_blank">mikael.holmen@ericsson.com</a>>> wrote:<br>
> <br>
> <br>
> <br>
> On 09/05/2018 09:56 AM, Sam McCall wrote:<br>
> > Sorry! r341451 should fix this, will keep an eye on the buildbots.<br>
> ><br>
> <br>
> Now it compiles with clang 3.6.0 but with gcc 5.4.0 it fails with<br>
> <br>
> /proj/bbi_twh/wh_bbi/x86_64-Linux2/bbigcc/<a href="http://1.5.4.0/crosscompiler/bin/g++" rel="noreferrer" target="_blank">1.5.4.0/crosscompiler/bin/g++</a><br>
> <<a href="http://1.5.4.0/crosscompiler/bin/g++" rel="noreferrer" target="_blank">http://1.5.4.0/crosscompiler/bin/g++</a>><br>
> -I/proj/bbi_twh/wh_bbi/x86_64-Linux2/bbilibxml2/1/include<br>
> -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS<br>
> -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS<br>
> -Itools/clang/tools/extra/clangd/global-symbol-builder<br>
> -I../tools/clang/tools/extra/clangd/global-symbol-builder<br>
> -I../tools/clang/include -Itools/clang/include -I/usr/include/libxml2<br>
> -Iinclude -I../include<br>
> -I../tools/clang/tools/extra/clangd/global-symbol-builder/..<br>
> -I/repo/app/valgrind/3.11.0/include -fPIC -fvisibility-inlines-hidden<br>
> -Werror=date-time -std=c++11 -Wall -Wextra -Wno-unused-parameter<br>
> -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic<br>
> -Wno-long-long -Wno-maybe-uninitialized -Wdelete-non-virtual-dtor<br>
> -Wno-comment -fdiagnostics-color -ffunction-sections -fdata-sections<br>
> -fno-common -Woverloaded-virtual -fno-strict-aliasing -O3 -UNDEBUG<br>
> -fno-exceptions -fno-rtti -MMD -MT<br>
> tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o<br>
> <br>
> -MF<br>
> tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o.d<br>
> <br>
> -o<br>
> tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o<br>
> <br>
> -c<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> In file included from<br>
> ../tools/clang/include/clang/Frontend/CommandLineSourceLoc.h:19:0,<br>
> from<br>
> ../tools/clang/include/clang/Frontend/FrontendOptions.h:13,<br>
> from<br>
> ../tools/clang/include/clang/Frontend/CompilerInvocation.h:19,<br>
> from<br>
> ../tools/clang/include/clang/Frontend/CompilerInstance.h:16,<br>
> from<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:22:<br>
> ../include/llvm/Support/CommandLine.h:606:52: error: invalid cast from<br>
> type 'llvm::cl::opt<clang::clangd::{anonymous}::Format>' to type 'int'<br>
> llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:67:16:<br>
> <br>
> note: in expansion of macro 'clEnumValN'<br>
> clEnumValN(Format::YAML, "yaml", "human-readable YAML<br>
> format"),<br>
> ^<br>
> ../include/llvm/Support/CommandLine.h:606:29: error: expected<br>
> primary-expression before '{' token<br>
> llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:67:16:<br>
> <br>
> note: in expansion of macro 'clEnumValN'<br>
> clEnumValN(Format::YAML, "yaml", "human-readable YAML<br>
> format"),<br>
> ^<br>
> ../include/llvm/Support/CommandLine.h:606:52: error: invalid cast from<br>
> type 'llvm::cl::opt<clang::clangd::{anonymous}::Format>' to type 'int'<br>
> llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:68:16:<br>
> <br>
> note: in expansion of macro 'clEnumValN'<br>
> clEnumValN(Format::Binary, "binary", "binary RIFF<br>
> format")),<br>
> ^<br>
> ../include/llvm/Support/CommandLine.h:606:29: error: expected<br>
> primary-expression before '{' token<br>
> llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:68:16:<br>
> <br>
> note: in expansion of macro 'clEnumValN'<br>
> clEnumValN(Format::Binary, "binary", "binary RIFF<br>
> format")),<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:69:27:<br>
> <br>
> error: 'Format' is not a class, namespace, or enumeration<br>
> llvm::cl::init(Format::YAML));<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:<br>
> <br>
> In function 'int main(int, const char**)':<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:276:23:<br>
> <br>
> error: 'clang::clangd::Format' is not a class, namespace, or enumeration<br>
> case clang::clangd::Format::YAML:<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:279:23:<br>
> <br>
> error: 'clang::clangd::Format' is not a class, namespace, or enumeration<br>
> case clang::clangd::Format::Binary: {<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10:<br>
> <br>
> warning: enumeration value 'YAML' not handled in switch [-Wswitch]<br>
> switch (clang::clangd::Format) {<br>
> ^<br>
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10:<br>
> <br>
> warning: enumeration value 'Binary' not handled in switch [-Wswitch]<br>
> <br>
> /Mikael<br>
> <br>
> > On Wed, Sep 5, 2018 at 8:46 AM Mikael Holmén<br>
> <<a href="mailto:mikael.holmen@ericsson.com" target="_blank">mikael.holmen@ericsson.com</a> <mailto:<a href="mailto:mikael.holmen@ericsson.com" target="_blank">mikael.holmen@ericsson.com</a>><br>
> > <mailto:<a href="mailto:mikael.holmen@ericsson.com" target="_blank">mikael.holmen@ericsson.com</a><br>
> <mailto:<a href="mailto:mikael.holmen@ericsson.com" target="_blank">mikael.holmen@ericsson.com</a>>>> wrote:<br>
> ><br>
> > Hi Sam,<br>
> ><br>
> > This doesn't compile for me. Both clang 3.6.0 and gcc 5.4.0<br>
> complain:<br>
> ><br>
> > [1/6] Building CXX object<br>
> > <br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o<br>
> > FAILED:<br>
> > <br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o<br>
> ><br>
> ><br>
> > /usr/bin/clang++ -march=corei7 -DGTEST_HAS_RTTI=0 -D_DEBUG<br>
> > -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS<br>
> > -D__STDC_LIMIT_MACROS -Itools/clang/tools/extra/clangd<br>
> > -I../tools/clang/tools/extra/clangd -I../tools/clang/include<br>
> > -Itools/clang/include -I/usr/include/libxml2 -Iinclude<br>
> -I../include<br>
> > -I/proj/flexasic/app/valgrind/3.11.0/include -fPIC<br>
> > -fvisibility-inlines-hidden -Werror -Werror=date-time<br>
> -std=c++11 -Wall<br>
> > -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual<br>
> > -Wmissing-field-initializers -pedantic -Wno-long-long<br>
> > -Wcovered-switch-default -Wnon-virtual-dtor<br>
> -Wdelete-non-virtual-dtor<br>
> > -Wstring-conversion -fdiagnostics-color -ffunction-sections<br>
> > -fdata-sections -fno-common -Woverloaded-virtual<br>
> -Wno-nested-anon-types<br>
> > -O3 -UNDEBUG -fno-exceptions -fno-rtti -MMD -MT<br>
> > <br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o<br>
> ><br>
> > -MF<br>
> > <br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o.d<br>
> ><br>
> > -o<br>
> > <br>
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o<br>
> ><br>
> > -c ../tools/clang/tools/extra/clangd/index/Serialization.cpp<br>
> > ../tools/clang/tools/extra/clangd/index/Serialization.cpp:154:10:<br>
> > error:<br>
> > no viable conversion from 'clang::clangd::(anonymous<br>
> > namespace)::StringTableIn' to 'Expected<clang::clangd::(anonymous<br>
> > namespace)::StringTableIn>'<br>
> > return Table;<br>
> > ^~~~~<br>
> > ../include/llvm/Support/Error.h:434:41: note: candidate<br>
> constructor<br>
> > (the<br>
> > implicit copy constructor) not viable: no known conversion from<br>
> > 'clang::clangd::(anonymous namespace)::StringTableIn' to 'const<br>
> > llvm::Expected<clang::clangd::(anonymous<br>
> namespace)::StringTableIn> &'<br>
> > for 1st argument<br>
> > template <class T> class LLVM_NODISCARD Expected {<br>
> > ^<br>
> > ../include/llvm/Support/Error.h:456:3: note: candidate<br>
> constructor not<br>
> > viable: no known conversion from 'clang::clangd::(anonymous<br>
> > namespace)::StringTableIn' to 'llvm::Error' for 1st argument<br>
> > Expected(Error Err)<br>
> > ^<br>
> > ../include/llvm/Support/Error.h:470:3: note: candidate<br>
> constructor not<br>
> > viable: no known conversion from 'clang::clangd::(anonymous<br>
> > namespace)::StringTableIn' to 'llvm::ErrorSuccess' for 1st<br>
> argument<br>
> > Expected(ErrorSuccess) = delete;<br>
> > ^<br>
> > ../include/llvm/Support/Error.h:488:3: note: candidate<br>
> constructor not<br>
> > viable: no known conversion from 'clang::clangd::(anonymous<br>
> > namespace)::StringTableIn' to<br>
> 'llvm::Expected<clang::clangd::(anonymous<br>
> > namespace)::StringTableIn> &&' for 1st argument<br>
> > Expected(Expected &&Other) {<br>
> moveConstruct(std::move(Other)); }<br>
> > ^<br>
> > ../include/llvm/Support/Error.h:476:36: note: candidate template<br>
> > ignored: disabled by 'enable_if' [with OtherT =<br>
> > clang::clangd::(anonymous namespace)::StringTableIn &]<br>
> > typename std::enable_if<std::is_convertible<OtherT,<br>
> > T>::value>::type<br>
> > ^<br>
> > ../include/llvm/Support/Error.h:493:3: note: candidate template<br>
> > ignored:<br>
> > could not match 'Expected<type-parameter-0-0>' against<br>
> > 'clang::clangd::(anonymous namespace)::StringTableIn'<br>
> > Expected(Expected<OtherT> &&Other,<br>
> > ^<br>
> > In file included from<br>
> > ../tools/clang/tools/extra/clangd/index/Serialization.cpp:9:<br>
> > In file included from<br>
> > ../tools/clang/tools/extra/clangd/index/Serialization.h:23:<br>
> > In file included from<br>
> > ../tools/clang/tools/extra/clangd/index/Index.h:13:<br>
> > In file included from<br>
> > ../tools/clang/include/clang/Index/IndexSymbol.h:14:<br>
> > In file included from<br>
> ../tools/clang/include/clang/Lex/MacroInfo.h:18:<br>
> > In file included from<br>
> ../tools/clang/include/clang/Lex/Token.h:17:<br>
> > In file included from<br>
> > ../tools/clang/include/clang/Basic/SourceLocation.h:19:<br>
> > In file included from ../include/llvm/ADT/StringRef.h:13:<br>
> > In file included from ../include/llvm/ADT/STLExtras.h:20:<br>
> > ../include/llvm/ADT/Optional.h:41:28: error: call to<br>
> implicitly-deleted<br>
> > copy constructor of 'clang::clangd::SymbolSlab'<br>
> > new (storage.buffer) T(*O.getPointer());<br>
> > ^ ~~~~~~~~~~~~~~~<br>
> > ../include/llvm/ADT/Optional.h:141:3: note: in instantiation<br>
> of member<br>
> > function<br>
> > <br>
> 'llvm::optional_detail::OptionalStorage<clang::clangd::SymbolSlab,<br>
> > false>::OptionalStorage' requested here<br>
> > Optional(const Optional &O) = default;<br>
> > ^<br>
> > <br>
> ../tools/clang/tools/extra/clangd/index/Serialization.cpp:325:10:<br>
> note:<br>
> > in instantiation of function template specialization<br>
> > <br>
> 'llvm::Expected<clang::clangd::IndexFileIn>::Expected<clang::clangd::IndexFileIn<br>
> ><br>
> > &>' requested here<br>
> > return Result;<br>
> > ^<br>
> > ../tools/clang/tools/extra/clangd/index/Index.h:324:26: note:<br>
> copy<br>
> > constructor of 'SymbolSlab' is implicitly deleted because<br>
> field 'Arena'<br>
> > has a deleted copy constructor<br>
> > llvm::BumpPtrAllocator Arena; // Owns Symbol data that the<br>
> > Symbols do<br>
> > not.<br>
> > ^<br>
> > ../include/llvm/Support/Allocator.h:157:3: note: copy<br>
> constructor is<br>
> > implicitly deleted because<br>
> 'BumpPtrAllocatorImpl<llvm::MallocAllocator,<br>
> > 4096, 4096>' has a user-declared move constructor<br>
> > BumpPtrAllocatorImpl(BumpPtrAllocatorImpl &&Old)<br>
> > ^<br>
> > 2 errors generated.<br>
> ><br>
> > Several buildbots fail the same way.<br>
> ><br>
> > /Mikael<br>
> ><br>
> > On 09/04/2018 06:16 PM, Sam McCall via cfe-commits wrote:<br>
> > > Author: sammccall<br>
> > > Date: Tue Sep 4 09:16:50 2018<br>
> > > New Revision: 341375<br>
> > ><br>
> > > URL: <a href="http://llvm.org/viewvc/llvm-project?rev=341375&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=341375&view=rev</a><br>
> > > Log:<br>
> > > [clangd] Define a compact binary serialization fomat for<br>
> symbol<br>
> > slab/index.<br>
> > ><br>
> > > Summary:<br>
> > > This is intended to replace the current YAML format for<br>
> general use.<br>
> > > It's ~10x more compact than YAML, and ~40% more compact than<br>
> > gzipped YAML:<br>
> > > llvmidx.riff = 20M, llvmidx.yaml = 272M,<br>
> llvmidx.yaml.gz = 32M<br>
> > > It's also simpler/faster to read and write.<br>
> > ><br>
> > > The format is a RIFF container (chunks of (type, size,<br>
> data)) with:<br>
> > > - a compressed string table<br>
> > > - simple binary encoding of symbols (with varints for<br>
> compactness)<br>
> > > It can be extended to include occurrences, Dex posting<br>
> lists, etc.<br>
> > ><br>
> > > There's no rich backwards-compatibility scheme, but a version<br>
> > number is included<br>
> > > so we can detect incompatible files and do ad-hoc back-compat.<br>
> > ><br>
> > > Alternatives considered:<br>
> > > - compressed YAML or JSON: bulky and slow to load<br>
> > > - llvm bitstream: confusing model and libraries are hard to<br>
> > use. My attempt<br>
> > > produced slightly larger files, and the code was<br>
> longer and<br>
> > slower.<br>
> > > - protobuf or similar: would be really nice (esp for<br>
> > back-compat) but the<br>
> > > dependency is a big hassle<br>
> > > - ad-hoc binary format without a container: it seems clear<br>
> > we're going<br>
> > > to add posting lists and occurrences here, and that<br>
> they will<br>
> > benefit<br>
> > > from sharing a string table. The container makes it<br>
> easy to debug<br>
> > > these pieces in isolation, and make them optional.<br>
> > ><br>
> > > Reviewers: ioeric<br>
> > ><br>
> > > Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, mgrang,<br>
> > arphaman, kadircet, cfe-commits<br>
> > ><br>
> > > Differential Revision: <a href="https://reviews.llvm.org/D51585" rel="noreferrer" target="_blank">https://reviews.llvm.org/D51585</a><br>
> > ><br>
> > > Added:<br>
> > > clang-tools-extra/trunk/clangd/RIFF.cpp<br>
> > > clang-tools-extra/trunk/clangd/RIFF.h<br>
> > > clang-tools-extra/trunk/clangd/index/Serialization.cpp<br>
> > > clang-tools-extra/trunk/clangd/index/Serialization.h<br>
> > > clang-tools-extra/trunk/unittests/clangd/RIFFTests.cpp<br>
> > > <br>
> clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp<br>
> > > Modified:<br>
> > > clang-tools-extra/trunk/clangd/CMakeLists.txt<br>
> > ><br>
> > <br>
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> > > clang-tools-extra/trunk/clangd/index/Index.cpp<br>
> > > clang-tools-extra/trunk/clangd/index/Index.h<br>
> > > clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp<br>
> > > clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp<br>
> > > clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt<br>
> > ><br>
> > clang-tools-extra/trunk/unittests/clangd/SymbolCollectorTests.cpp<br>
> > ><br>
> > > Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt<br>
> > > URL:<br>
> ><br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> > ><br>
> > <br>
> ==============================================================================<br>
> > > --- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)<br>
> > > +++ clang-tools-extra/trunk/clangd/CMakeLists.txt Tue Sep 4<br>
> > 09:16:50 2018<br>
> > > @@ -29,6 +29,7 @@ add_clang_library(clangDaemon<br>
> > > Protocol.cpp<br>
> > > ProtocolHandlers.cpp<br>
> > > Quality.cpp<br>
> > > + RIFF.cpp<br>
> > > SourceCode.cpp<br>
> > > Threading.cpp<br>
> > > Trace.cpp<br>
> > > @@ -41,6 +42,7 @@ add_clang_library(clangDaemon<br>
> > > index/Index.cpp<br>
> > > index/MemIndex.cpp<br>
> > > index/Merge.cpp<br>
> > > + index/Serialization.cpp<br>
> > > index/SymbolCollector.cpp<br>
> > > index/SymbolYAML.cpp<br>
> > ><br>
> > ><br>
> > > Added: clang-tools-extra/trunk/clangd/RIFF.cpp<br>
> > > URL:<br>
> ><br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.cpp?rev=341375&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.cpp?rev=341375&view=auto</a><br>
> > ><br>
> > <br>
> ==============================================================================<br>
> > > --- clang-tools-extra/trunk/clangd/RIFF.cpp (added)<br>
> > > +++ clang-tools-extra/trunk/clangd/RIFF.cpp Tue Sep 4<br>
> 09:16:50 2018<br>
> > > @@ -0,0 +1,88 @@<br>
> > > +//===--- RIFF.cpp - Binary container file format<br>
> > --------------------------===//<br>
> > > +//<br>
> > > +// The LLVM Compiler Infrastructure<br>
> > > +//<br>
> > > +// This file is distributed under the University of Illinois<br>
> > Open Source<br>
> > > +// License. See LICENSE.TXT for details.<br>
> > > +//<br>
> > ><br>
> > <br>
> +//===----------------------------------------------------------------------===//<br>
> > > +<br>
> > > +#include "RIFF.h"<br>
> > > +#include "llvm/Support/Endian.h"<br>
> > > +<br>
> > > +using namespace llvm;<br>
> > > +namespace clang {<br>
> > > +namespace clangd {<br>
> > > +namespace riff {<br>
> > > +<br>
> > > +static Error makeError(const char *Msg) {<br>
> > > + return createStringError(inconvertibleErrorCode(), Msg);<br>
> > > +}<br>
> > > +<br>
> > > +Expected<Chunk> readChunk(StringRef &Stream) {<br>
> > > + if (Stream.size() < 8)<br>
> > > + return makeError("incomplete chunk header");<br>
> > > + Chunk C;<br>
> > > + std::copy(Stream.begin(), Stream.begin() + 4,<br>
> C.ID.begin());<br>
> > > + Stream = Stream.drop_front(4);<br>
> > > + uint32_t Len =<br>
> > support::endian::read32le(Stream.take_front(4).begin());<br>
> > > + Stream = Stream.drop_front(4);<br>
> > > + if (Stream.size() < Len)<br>
> > > + return makeError("truncated chunk");<br>
> > > + C.Data = Stream.take_front(Len);<br>
> > > + Stream = Stream.drop_front(Len);<br>
> > > + if (Len % 2 & !Stream.empty()) { // Skip padding byte.<br>
> > > + if (Stream.front())<br>
> > > + return makeError("nonzero padding byte");<br>
> > > + Stream = Stream.drop_front();<br>
> > > + }<br>
> > > + return C;<br>
> > > +};<br>
> > > +<br>
> > > +raw_ostream &operator<<(raw_ostream &OS, const Chunk &C) {<br>
> > > + OS.write(C.ID.begin(), C.ID.size());<br>
> > > + char Size[4];<br>
> > > + llvm::support::endian::write32le(Size, C.Data.size());<br>
> > > + OS.write(Size, sizeof(Size));<br>
> > > + OS << C.Data;<br>
> > > + if (C.Data.size() % 2)<br>
> > > + OS.write(0);<br>
> > > + return OS;<br>
> > > +}<br>
> > > +<br>
> > > +llvm::Expected<File> readFile(llvm::StringRef Stream) {<br>
> > > + auto RIFF = readChunk(Stream);<br>
> > > + if (!RIFF)<br>
> > > + return RIFF.takeError();<br>
> > > + if (RIFF->ID != fourCC("RIFF"))<br>
> > > + return makeError("not a RIFF container");<br>
> > > + if (RIFF->Data.size() < 4)<br>
> > > + return makeError("RIFF chunk too short");<br>
> > > + File F;<br>
> > > + std::copy(RIFF->Data.begin(), RIFF->Data.begin() + 4,<br>
> > F.Type.begin());<br>
> > > + for (llvm::StringRef Body = RIFF->Data.drop_front(4);<br>
> > !Body.empty();)<br>
> > > + if (auto Chunk = readChunk(Body)) {<br>
> > > + F.Chunks.push_back(*Chunk);<br>
> > > + } else<br>
> > > + return Chunk.takeError();<br>
> > > + return F;<br>
> > > +}<br>
> > > +<br>
> > > +raw_ostream &operator<<(raw_ostream &OS, const File &F) {<br>
> > > + // To avoid copies, we serialize the outer RIFF chunk<br>
> "by hand".<br>
> > > + size_t DataLen = 4; // Predict length of RIFF chunk data.<br>
> > > + for (const auto &C : F.Chunks)<br>
> > > + DataLen += 4 + 4 + C.Data.size() + (C.Data.size() % 2);<br>
> > > + OS << "RIFF";<br>
> > > + char Size[4];<br>
> > > + llvm::support::endian::write32le(Size, DataLen);<br>
> > > + OS.write(Size, sizeof(Size));<br>
> > > + OS.write(F.Type.begin(), F.Type.size());<br>
> > > + for (const auto &C : F.Chunks)<br>
> > > + OS << C;<br>
> > > + return OS;<br>
> > > +}<br>
> > > +<br>
> > > +} // namespace riff<br>
> > > +} // namespace clangd<br>
> > > +} // namespace clang<br>
> > ><br>
> > > Added: clang-tools-extra/trunk/clangd/RIFF.h<br>
> > > URL:<br>
> ><br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.h?rev=341375&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.h?rev=341375&view=auto</a><br>
> > ><br>
> > <br>
> ==============================================================================<br>
> > > --- clang-tools-extra/trunk/clangd/RIFF.h (added)<br>
> > > +++ clang-tools-extra/trunk/clangd/RIFF.h Tue Sep 4<br>
> 09:16:50 2018<br>
> > > @@ -0,0 +1,81 @@<br>
> > > +//===--- RIFF.h - Binary container file format<br>
> > -------------------*- C++-*-===//<br>
> > > +//<br>
> > > +// The LLVM Compiler Infrastructure<br>
> > > +//<br>
> > > +// This file is distributed under the University of Illinois<br>
> > Open Source<br>
> > > +// License. See LICENSE.TXT for details.<br>
> > > +//<br>
> > ><br>
> > <br>
> +//===----------------------------------------------------------------------===//<br>
> > > +//<br>
> > > +// Tools for reading and writing data in RIFF containers.<br>
> > > +//<br>
> > > +// A chunk consists of:<br>
> > > +// - ID : char[4]<br>
> > > +// - Length : uint32<br>
> > > +// - Data : byte[Length]<br>
> > > +// - Padding : byte[Length % 2]<br>
> > > +// The semantics of a chunk's Data are determined by its ID.<br>
> > > +// The format makes it easy to skip over uninteresting or<br>
> > unknown chunks.<br>
> > > +//<br>
> > > +// A RIFF file is a single chunk with ID "RIFF". Its Data is:<br>
> > > +// - Type : char[4]<br>
> > > +// - Chunks : chunk[]<br>
> > > +//<br>
> > > +// This means that a RIFF file consists of:<br>
> > > +// - "RIFF" : char[4]<br>
> > > +// - File length - 8 : uint32<br>
> > > +// - File type : char[4]<br>
> > > +// - Chunks : chunk[]<br>
> > > +//<br>
> > ><br>
> > <br>
> +//===----------------------------------------------------------------------===//<br>
> > > +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_RIFF_H<br>
> > > +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_RIFF_H<br>
> > > +#include "llvm/ADT/StringRef.h"<br>
> > > +#include "llvm/Support/Error.h"<br>
> > > +#include "llvm/Support/ScopedPrinter.h"<br>
> > > +#include <array><br>
> > > +<br>
> > > +namespace clang {<br>
> > > +namespace clangd {<br>
> > > +namespace riff {<br>
> > > +<br>
> > > +// A FourCC identifies a chunk in a file, or the type of file<br>
> > itself.<br>
> > > +using FourCC = std::array<char, 4>;<br>
> > > +// Get a FourCC from a string literal, e.g. fourCC("RIFF").<br>
> > > +inline constexpr FourCC fourCC(const char (&Literal)[5]) {<br>
> > > + return FourCC{{Literal[0], Literal[1], Literal[2],<br>
> Literal[3]}};<br>
> > > +}<br>
> > > +// A chunk is a section in a RIFF container.<br>
> > > +struct Chunk {<br>
> > > + FourCC ID;<br>
> > > + llvm::StringRef Data;<br>
> > > +};<br>
> > > +inline bool operator==(const Chunk &L, const Chunk &R) {<br>
> > > + return std::tie(<a href="http://L.ID" rel="noreferrer" target="_blank">L.ID</a> <<a href="http://L.ID" rel="noreferrer" target="_blank">http://L.ID</a>> <<a href="http://L.ID" rel="noreferrer" target="_blank">http://L.ID</a>>,<br>
> L.Data) == std::tie(<a href="http://R.ID" rel="noreferrer" target="_blank">R.ID</a> <<a href="http://R.ID" rel="noreferrer" target="_blank">http://R.ID</a>><br>
> > <<a href="http://R.ID" rel="noreferrer" target="_blank">http://R.ID</a>>, R.Data);<br>
> > > +}<br>
> > > +// A File is a RIFF container, which is a typed chunk<br>
> sequence.<br>
> > > +struct File {<br>
> > > + FourCC Type;<br>
> > > + std::vector<Chunk> Chunks;<br>
> > > +};<br>
> > > +inline bool operator==(const File &L, const File &R) {<br>
> > > + return std::tie(L.Type, L.Chunks) == std::tie(R.Type,<br>
> R.Chunks);<br>
> > > +}<br>
> > > +<br>
> > > +// Reads a single chunk from the start of Stream.<br>
> > > +// Stream is updated to exclude the consumed chunk.<br>
> > > +llvm::Expected<Chunk> readChunk(llvm::StringRef &Stream);<br>
> > > +<br>
> > > +// Serialize a single chunk to OS.<br>
> > > +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,<br>
> const Chunk &);<br>
> > > +<br>
> > > +// Parses a RIFF file consisting of a single RIFF chunk.<br>
> > > +llvm::Expected<File> readFile(llvm::StringRef Stream);<br>
> > > +<br>
> > > +// Serialize a RIFF file (i.e. a single RIFF chunk) to OS.<br>
> > > +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,<br>
> const File &);<br>
> > > +<br>
> > > +} // namespace riff<br>
> > > +} // namespace clangd<br>
> > > +} // namespace clang<br>
> > > +#endif<br>
> > ><br>
> > > Modified:<br>
> > <br>
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> > > URL:<br>
> ><br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> > ><br>
> > <br>
> ==============================================================================<br>
> > > ---<br>
> > <br>
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> > (original)<br>
> > > +++<br>
> > <br>
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp<br>
> > Tue Sep 4 09:16:50 2018<br>
> > > @@ -7,15 +7,16 @@<br>
> > > //<br>
> > ><br>
> > <br>
> //===----------------------------------------------------------------------===//<br>
> > > //<br>
> > > -// GlobalSymbolBuilder is a tool to generate YAML-format<br>
> symbols<br>
> > across the<br>
> > > -// whole project. This tools is for **experimental**<br>
> only. Don't<br>
> > use it in<br>
> > > -// production code.<br>
> > > +// GlobalSymbolBuilder is a tool to extract symbols from<br>
> a whole<br>
> > project.<br>
> > > +// This tool is **experimental** only. Don't use it in<br>
> > production code.<br>
> > > //<br>
> > ><br>
> > <br>
> //===----------------------------------------------------------------------===//<br>
> > ><br>
> > > +#include "RIFF.h"<br>
> > > #include "index/CanonicalIncludes.h"<br>
> > > #include "index/Index.h"<br>
> > > #include "index/Merge.h"<br>
> > > +#include "index/Serialization.h"<br>
> > > #include "index/SymbolCollector.h"<br>
> > > #include "index/SymbolYAML.h"<br>
> > > #include "clang/Frontend/CompilerInstance.h"<br>
> > > @@ -59,6 +60,14 @@ static llvm::cl::opt<bool> MergeOnTheFly<br>
> > > "MapReduce."),<br>
> > > llvm::cl::init(true), llvm::cl::Hidden);<br>
> > ><br>
> > > +enum class Format { YAML, Binary };<br>
> > > +static llvm::cl::opt<Format><br>
> > > + Format("format", llvm::cl::desc("Format of the index<br>
> to be<br>
> > written"),<br>
> > > + llvm::cl::values(<br>
> > > + clEnumValN(Format::YAML, "yaml",<br>
> "human-readable<br>
> > YAML format"),<br>
> > > + clEnumValN(Format::Binary, "binary",<br>
> "binary RIFF<br>
> > format")),<br>
> > > + llvm::cl::init(Format::YAML));<br>
> > > +<br>
> > > /// Responsible for aggregating symbols from each processed<br>
> > file and producing<br>
> > > /// the final results. All methods in this class must be<br>
> > thread-safe,<br>
> > > /// 'consumeSymbols' may be called from multiple threads.<br>
> > > @@ -210,8 +219,8 @@ int main(int argc, const char **argv) {<br>
> > > llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);<br>
> > ><br>
> > > const char *Overview = R"(<br>
> > > - This is an **experimental** tool to generate YAML-format<br>
> > project-wide symbols<br>
> > > - for clangd (global code completion). It would be<br>
> changed and<br>
> > deprecated<br>
> > > + This is an **experimental** tool to extract symbols from a<br>
> > whole project<br>
> > > + for clangd (global code completion). It will be changed and<br>
> > deprecated<br>
> > > eventually. Don't use it in production code!<br>
> > ><br>
> > > Example usage for building index for the whole project<br>
> using<br>
> > CMake compile<br>
> > > @@ -262,7 +271,16 @@ int main(int argc, const char **argv) {<br>
> > > }<br>
> > > // Reduce phase: combine symbols with the same IDs.<br>
> > > auto UniqueSymbols = Consumer->mergeResults();<br>
> > > - // Output phase: emit YAML for result symbols.<br>
> > > - SymbolsToYAML(UniqueSymbols, llvm::outs());<br>
> > > + // Output phase: emit result symbols.<br>
> > > + switch (clang::clangd::Format) {<br>
> > > + case clang::clangd::Format::YAML:<br>
> > > + SymbolsToYAML(UniqueSymbols, llvm::outs());<br>
> > > + break;<br>
> > > + case clang::clangd::Format::Binary: {<br>
> > > + clang::clangd::IndexFileOut Out;<br>
> > > + Out.Symbols = &UniqueSymbols;<br>
> > > + llvm::outs() << Out;<br>
> > > + }<br>
> > > + }<br>
> > > return 0;<br>
> > > }<br>
> > ><br>
> > > Modified: clang-tools-extra/trunk/clangd/index/Index.cpp<br>
> > > URL:<br>
> ><br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> > ><br>
> > <br>
> ==============================================================================<br>
> > > --- clang-tools-extra/trunk/clangd/index/Index.cpp (original)<br>
> > > +++ clang-tools-extra/trunk/clangd/index/Index.cpp Tue Sep 4<br>
> > 09:16:50 2018<br>
> > > @@ -10,6 +10,7 @@<br>
> > > #include "Index.h"<br>
> > > #include "llvm/ADT/StringExtras.h"<br>
> > > #include "llvm/ADT/StringRef.h"<br>
> > > +#include "llvm/Support/Error.h"<br>
> > > #include "llvm/Support/SHA1.h"<br>
> > > #include "llvm/Support/raw_ostream.h"<br>
> > ><br>
> > > @@ -28,21 +29,20 @@ SymbolID::SymbolID(StringRef USR)<br>
> > > : HashValue(SHA1::hash(arrayRefFromStringRef(USR))) {}<br>
> > ><br>
> > > raw_ostream &operator<<(raw_ostream &OS, const SymbolID<br>
> &ID) {<br>
> > > - OS << toHex(toStringRef(ID.HashValue));<br>
> > > - return OS;<br>
> > > + return OS << toHex(ID.raw());<br>
> > > }<br>
> > ><br>
> > > -std::string SymbolID::str() const {<br>
> > > - std::string ID;<br>
> > > - llvm::raw_string_ostream OS(ID);<br>
> > > - OS << *this;<br>
> > > - return OS.str();<br>
> > > +SymbolID SymbolID::fromRaw(llvm::StringRef Raw) {<br>
> > > + SymbolID ID;<br>
> > > + assert(Raw.size() == RawSize);<br>
> > > + memcpy(ID.HashValue.data(), Raw.data(), RawSize);<br>
> > > + return ID;<br>
> > > }<br>
> > ><br>
> > > +std::string SymbolID::str() const { return toHex(raw()); }<br>
> > > +<br>
> > > void operator>>(StringRef Str, SymbolID &ID) {<br>
> > > - std::string HexString = fromHex(Str);<br>
> > > - assert(HexString.size() == ID.HashValue.size());<br>
> > > - std::copy(HexString.begin(), HexString.end(),<br>
> > ID.HashValue.begin());<br>
> > > + ID = SymbolID::fromRaw(fromHex(Str));<br>
> > > }<br>
> > ><br>
> > > raw_ostream &operator<<(raw_ostream &OS, SymbolOrigin O) {<br>
> > > @@ -78,34 +78,18 @@ SymbolSlab::const_iterator SymbolSlab::f<br>
> > > }<br>
> > ><br>
> > > // Copy the underlying data of the symbol into the owned<br>
> arena.<br>
> > > -static void own(Symbol &S, llvm::UniqueStringSaver &Strings,<br>
> > > - BumpPtrAllocator &Arena) {<br>
> > > - // Intern replaces V with a reference to the same<br>
> string owned<br>
> > by the arena.<br>
> > > - auto Intern = [&](StringRef &V) { V = Strings.save(V); };<br>
> > > -<br>
> > > - // We need to copy every StringRef field onto the arena.<br>
> > > - Intern(S.Name);<br>
> > > - Intern(S.Scope);<br>
> > > - Intern(S.CanonicalDeclaration.FileURI);<br>
> > > - Intern(S.Definition.FileURI);<br>
> > > -<br>
> > > - Intern(S.Signature);<br>
> > > - Intern(S.CompletionSnippetSuffix);<br>
> > > -<br>
> > > - Intern(S.Documentation);<br>
> > > - Intern(S.ReturnType);<br>
> > > - for (auto &I : S.IncludeHeaders)<br>
> > > - Intern(I.IncludeHeader);<br>
> > > +static void own(Symbol &S, llvm::UniqueStringSaver<br>
> &Strings) {<br>
> > > + visitStrings(S, [&](StringRef &V) { V =<br>
> Strings.save(V); });<br>
> > > }<br>
> > ><br>
> > > void SymbolSlab::Builder::insert(const Symbol &S) {<br>
> > > auto R = SymbolIndex.try_emplace(<a href="http://S.ID" rel="noreferrer" target="_blank">S.ID</a> <<a href="http://S.ID" rel="noreferrer" target="_blank">http://S.ID</a>><br>
> <<a href="http://S.ID" rel="noreferrer" target="_blank">http://S.ID</a>>,<br>
> > Symbols.size());<br>
> > > if (R.second) {<br>
> > > Symbols.push_back(S);<br>
> > > - own(Symbols.back(), UniqueStrings, Arena);<br>
> > > + own(Symbols.back(), UniqueStrings);<br>
> > > } else {<br>
> > > auto &Copy = Symbols[R.first->second] = S;<br>
> > > - own(Copy, UniqueStrings, Arena);<br>
> > > + own(Copy, UniqueStrings);<br>
> > > }<br>
> > > }<br>
> > ><br>
> > > @@ -118,7 +102,7 @@ SymbolSlab SymbolSlab::Builder::build()<br>
> > > BumpPtrAllocator NewArena;<br>
> > > llvm::UniqueStringSaver Strings(NewArena);<br>
> > > for (auto &S : Symbols)<br>
> > > - own(S, Strings, NewArena);<br>
> > > + own(S, Strings);<br>
> > > return SymbolSlab(std::move(NewArena),<br>
> std::move(Symbols));<br>
> > > }<br>
> > ><br>
> > ><br>
> > > Modified: clang-tools-extra/trunk/clangd/index/Index.h<br>
> > > URL:<br>
> ><br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=341375&r1=341374&r2=341375&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=341375&r1=341374&r2=341375&view=diff</a><br>
> > ><br>
> > <br>
> ==============================================================================<br>
> > > --- clang-tools-extra/trunk/clangd/index/Index.h (original)<br>
> > > +++ clang-tools-extra/trunk/clangd/index/Index.h Tue Sep 4<br>
> > 09:16:50 2018<br>
> > > @@ -84,26 +84,28 @@ public:<br>
> > > return HashValue < Sym.HashValue;<br>
> > > }<br>
> > ><br>
> > > + constexpr static size_t RawSize = 20;<br>
> > > + llvm::StringRef raw() const {<br>
> > > + return StringRef(reinterpret_cast<const char<br>
> > *>(HashValue.data()), RawSize);<br>
> > > + }<br>
> > > + static SymbolID fromRaw(llvm::StringRef);<br>
> > > // Returns a 40-bytes hex encoded string.<br>
> > > std::string str() const;<br>
> > ><br>
> > > private:<br>
> > > - static constexpr unsigned HashByteLength = 20;<br>
> > > -<br>
> > > - friend llvm::hash_code hash_value(const SymbolID &ID) {<br>
> > > - // We already have a good hash, just return the first<br>
> bytes.<br>
> > > - static_assert(sizeof(size_t) <= HashByteLength, "size_t<br>
> > longer than SHA1!");<br>
> > > - size_t Result;<br>
> > > - memcpy(&Result, ID.HashValue.data(), sizeof(size_t));<br>
> > > - return llvm::hash_code(Result);<br>
> > > - }<br>
> > > - friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,<br>
> > > - const SymbolID &ID);<br>
> > > friend void operator>>(llvm::StringRef Str, SymbolID &ID);<br>
> > ><br>
> > > - std::array<uint8_t, HashByteLength> HashValue;<br>
> > > + std::array<uint8_t, RawSize> HashValue;<br>
> > > };<br>
> > ><br>
> > > +inline llvm::hash_code hash_value(const SymbolID &ID) {<br>
> > > + // We already have a good hash, just return the first<br>
> bytes.<br>
> > > + assert(sizeof(size_t) <= SymbolID::RawSize && "size_t<br>
> longer<br>
> > than SHA1!");<br>
> > > + size_t Result;<br>
> > > + memcpy(&Result, ID.raw().data(), sizeof(size_t));<br>
> > > + return llvm::hash_code(Result);<br>
> > > +}<br>
> > > +<br>
> > > // Write SymbolID into the given stream. SymbolID is<br>
> encoded as<br>
> > a 40-bytes<br>
> > > // hex string.<br>
> > > llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const<br>
> > SymbolID &ID);<br>
> > > @@ -246,6 +248,21 @@ struct Symbol {<br>
> > > };<br>
> > > llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const<br>
> > Symbol &S);<br>
> > ><br>
> > > +// Invokes Callback with each StringRef& contained in the<br>
> Symbol.<br>
> > > +// Useful for deduplicating backing strings.<br>
> > > +template <typename Callback> void visitStrings(Symbol &S,<br>
> const<br>
> > Callback &CB) {<br>
> > > + CB(S.Name);<br>
> > > + CB(S.Scope);<br>
> > > + CB(S.CanonicalDeclaration.FileURI);<br>
> > > + CB(S.Definition.FileURI);<br>
> > > + CB(S.Signature);<br>
> > > + CB(S.CompletionSnippetSuffix);<br>
> > > + CB(S.Documentation);<br>
> > > + CB(S.ReturnType);<br>
> > > + for (auto &Include : S.IncludeHeaders)<br>
> > > + CB(Include.IncludeHeader);<br>
> > > +}<br>
> > > +<br>
> > > // Computes query-independent quality score for a Symbol.<br>
> > > // This currently falls in the range [1, ln(#indexed<br>
> documents)].<br>
> > > // FIXME: this should probably be split into symbol -><br>
> signals<br>
> > ><br>
> > > Added: clang-tools-extra/trunk/clangd/index/Serialization.cpp<br>
> > > URL:<br>
> ><br>
> <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=341375&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=341375&view=auto</a><br>
> > ><br>
> > <br>
> ==============================================================================<br>
> > > --- clang-tools-extra/trunk/clangd/index/Serialization.cpp<br>
> (added)<br>
> > > +++ clang-tools-extra/trunk/clangd/index/Serialization.cpp Tue<br>
> > Sep 4 09:16:50 2018<br>
> > > @@ -0,0 +1,366 @@<br>
> > > +//===-- Serialization.cpp - Binary serialization of index<br>
> data<br>
> > ------------===//<br>
> > > +//<br>
> > > +// The LLVM Compiler Infrastructure<br>
> > > +//<br>
> > > +// This file is distributed under the University of Illinois<br>
> > Open Source<br>
> > > +// License. See LICENSE.TXT for details.<br>
> > > +//<br>
> > ><br>
> > <br>
> +//===----------------------------------------------------------------------===//<br>
> > > +#include "Serialization.h"<br>
> > > +#include "../RIFF.h"<br>
> > > +#include "llvm/Support/Compression.h"<br>
> > > +#include "llvm/Support/Endian.h"<br>
> > > +#include "llvm/Support/Error.h"<br>
> > > +<br>
> > > +using namespace llvm;<br>
> > > +namespace clang {<br>
> > > +namespace clangd {<br>
> > > +namespace {<br>
> > > +Error makeError(const Twine &Msg) {<br>
> > > + return make_error<StringError>(Msg,<br>
> inconvertibleErrorCode());<br>
> > > +}<br>
> > > +<br>
> > > +// IO PRIMITIVES<br>
> > > +// We use little-endian 32 bit ints, sometimes with<br>
> > variable-length encoding.<br>
> > > +<br>
> > > +StringRef consume(StringRef &Data, int N) {<br>
> > > + StringRef Ret = Data.take_front(N);<br>
> > > + Data = Data.drop_front(N);<br>
> > > + return Ret;<br>
> > > +}<br>
> > > +<br>
> > > +uint8_t consume8(StringRef &Data) {<br>
> > > + uint8_t Ret = Data.front();<br>
> > > + Data = Data.drop_front();<br>
> > > + return Ret;<br>
> > > +}<br>
> > > +<br>
> > > +uint32_t consume32(StringRef &Data) {<br>
> > > + auto Ret = support::endian::read32le(Data.bytes_begin());<br>
> > > + Data = Data.drop_front(4);<br>
> > > + return Ret;<br>
> > > +}<br>
> > > +<br>
> > > +void write32(uint32_t I, raw_ostream &OS) {<br>
> > > + char buf[4];<br>
> > > + support::endian::write32le(buf, I);<br>
> > > + OS.write(buf, sizeof(buf));<br>
> > > +}<br>
> > > +<br>
> > > +// Variable-length int encoding (varint) uses the bottom<br>
> 7 bits<br>
> > of each byte<br>
> > > +// to encode the number, and the top bit to indicate whether<br>
> > more bytes follow.<br>
> > > +// e.g. 9a 2f means [0x1a and keep reading, 0x2f and stop].<br>
> > > +// This represents 0x1a | 0x2f<<7 = 6042.<br>
> > > +// A 32-bit integer takes 1-5 bytes to encode; small<br>
> numbers are<br>
> > more compact.<br>
> > > +void writeVar(uint32_t I, raw_ostream &OS) {<br>
> > > + constexpr static uint8_t More = 1 << 7;<br>
> > > + if (LLVM_LIKELY(I < 1 << 7)) {<br>
> > > + OS.write(I);<br>
> > > + return;<br>
> > > + }<br>
> > > + for (;;) {<br>
> > > + OS.write(I | More);<br>
> > > + I >>= 7;<br>
> > > + if (I < 1 << 7) {<br>
> > > + OS.write(I);<br>
> > > + return;<br>
> > > + }<br>
> > > + }<br>
> > > +}<br>
> > > +<br>
> > > +uint32_t consumeVar(StringRef &Data) {<br>
> > > + constexpr static uint8_t More = 1 << 7;<br>
> > > + uint8_t B = consume8(Data);<br>
> > </blockquote></div>