[clang-tools-extra] r341375 - [clangd] Define a compact binary serialization fomat for symbol slab/index.
Mikael Holmén via cfe-commits
cfe-commits at lists.llvm.org
Wed Sep 5 22:57:00 PDT 2018
On 09/05/2018 03:26 PM, Sam McCall wrote:
> Ah, thanks! Fixed in r341467.
>
> I'd guess this was a GCC bug, because GCC >=6 accepts it:
> https://godbolt.org/z/Yb5gBT
>
Yep works now. Thanks!
/Mikael
> On Wed, Sep 5, 2018 at 2:21 PM Mikael Holmén <mikael.holmen at ericsson.com
> <mailto:mikael.holmen at ericsson.com>> wrote:
>
>
>
> On 09/05/2018 12:41 PM, Sam McCall wrote:
> > Thanks. Unclear to me whether it's the enum class or the anonymous
> > namespace that's triggering this (I believe) compiler bug,
> but r341459
> > may help...
>
> Still doesn't work.
>
> In file included from
> ../tools/clang/include/clang/Frontend/CommandLineSourceLoc.h:19:0,
> from
> ../tools/clang/include/clang/Frontend/FrontendOptions.h:13,
> from
> ../tools/clang/include/clang/Frontend/CompilerInvocation.h:19,
> from
> ../tools/clang/include/clang/Frontend/CompilerInstance.h:16,
> from
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:22:
> ../include/llvm/Support/CommandLine.h:606:29: error: expected
> primary-expression before '{' token
> llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }
> ^
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:67:16:
>
> note: in expansion of macro 'clEnumValN'
> clEnumValN(Format::YAML, "yaml", "human-readable YAML
> format"),
> ^
> ../include/llvm/Support/CommandLine.h:606:29: error: expected
> primary-expression before '{' token
> llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }
> ^
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:68:16:
>
> note: in expansion of macro 'clEnumValN'
> clEnumValN(Format::Binary, "binary", "binary RIFF
> format")),
> ^
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:69:27:
>
> error: 'Format' is not a class, namespace, or enumeration
> llvm::cl::init(Format::YAML));
> ^
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:
>
> In function 'int main(int, const char**)':
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:276:23:
>
> error: 'clang::clangd::Format' is not a class, namespace, or enumeration
> case clang::clangd::Format::YAML:
> ^
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:279:23:
>
> error: 'clang::clangd::Format' is not a class, namespace, or enumeration
> case clang::clangd::Format::Binary: {
> ^
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10:
>
> warning: enumeration value 'YAML' not handled in switch [-Wswitch]
> switch (clang::clangd::Format) {
> ^
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10:
>
> warning: enumeration value 'Binary' not handled in switch [-Wswitch]
>
> Changing the name of the enum from Format to Formats (so the name isn't
> the same as the variable) makes it compile.
>
> I.e:
>
> @@ -58,17 +58,17 @@ static llvm::cl::opt<bool> MergeOnTheFly(
> "usage and an almost instant reduce stage. Optimal for
> running
> as a "
> "standalone tool, but cannot be used with multi-process
> executors like "
> "MapReduce."),
> llvm::cl::init(true), llvm::cl::Hidden);
>
> -enum Format { YAML, Binary };
> -static llvm::cl::opt<Format>
> +enum Formats { YAML, Binary };
> +static llvm::cl::opt<Formats>
> Format("format", llvm::cl::desc("Format of the index to be
> written"),
> llvm::cl::values(
> - clEnumValN(Format::YAML, "yaml", "human-readable YAML
> format"),
> - clEnumValN(Format::Binary, "binary", "binary RIFF
> format")),
> - llvm::cl::init(Format::YAML));
> + clEnumValN(Formats::YAML, "yaml", "human-readable YAML
> format"),
> + clEnumValN(Formats::Binary, "binary", "binary RIFF
> format")),
> + llvm::cl::init(Formats::YAML));
>
> /// Responsible for aggregating symbols from each processed file and
> producing
> /// the final results. All methods in this class must be thread-safe,
> /// 'consumeSymbols' may be called from multiple threads.
> class SymbolsConsumer {
> @@ -271,14 +271,14 @@ int main(int argc, const char **argv) {
> }
> // Reduce phase: combine symbols with the same IDs.
> auto UniqueSymbols = Consumer->mergeResults();
> // Output phase: emit result symbols.
> switch (clang::clangd::Format) {
> - case clang::clangd::Format::YAML:
> + case clang::clangd::Formats::YAML:
> SymbolsToYAML(UniqueSymbols, llvm::outs());
> break;
> - case clang::clangd::Format::Binary: {
> + case clang::clangd::Formats::Binary: {
> clang::clangd::IndexFileOut Out;
> Out.Symbols = &UniqueSymbols;
> llvm::outs() << Out;
> }
> }
>
> seems to compile with gcc 5.4.0.
>
> I've no idea if this is a gcc bug or if it's a bug in clang to not also
> complain about it.
>
> /Mikael
>
> >
> > On Wed, Sep 5, 2018 at 11:05 AM Mikael Holmén
> > <mikael.holmen at ericsson.com <mailto:mikael.holmen at ericsson.com>
> <mailto:mikael.holmen at ericsson.com
> <mailto:mikael.holmen at ericsson.com>>> wrote:
> >
> >
> >
> > On 09/05/2018 09:56 AM, Sam McCall wrote:
> > > Sorry! r341451 should fix this, will keep an eye on the
> buildbots.
> > >
> >
> > Now it compiles with clang 3.6.0 but with gcc 5.4.0 it fails with
> >
> >
> /proj/bbi_twh/wh_bbi/x86_64-Linux2/bbigcc/1.5.4.0/crosscompiler/bin/g++ <http://1.5.4.0/crosscompiler/bin/g++>
> > <http://1.5.4.0/crosscompiler/bin/g++>
> > -I/proj/bbi_twh/wh_bbi/x86_64-Linux2/bbilibxml2/1/include
> > -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GNU_SOURCE
> -D__STDC_CONSTANT_MACROS
> > -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS
> > -Itools/clang/tools/extra/clangd/global-symbol-builder
> > -I../tools/clang/tools/extra/clangd/global-symbol-builder
> > -I../tools/clang/include -Itools/clang/include
> -I/usr/include/libxml2
> > -Iinclude -I../include
> > -I../tools/clang/tools/extra/clangd/global-symbol-builder/..
> > -I/repo/app/valgrind/3.11.0/include -fPIC
> -fvisibility-inlines-hidden
> > -Werror=date-time -std=c++11 -Wall -Wextra -Wno-unused-parameter
> > -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers
> -pedantic
> > -Wno-long-long -Wno-maybe-uninitialized -Wdelete-non-virtual-dtor
> > -Wno-comment -fdiagnostics-color -ffunction-sections
> -fdata-sections
> > -fno-common -Woverloaded-virtual -fno-strict-aliasing -O3
> -UNDEBUG
> > -fno-exceptions -fno-rtti -MMD -MT
> >
> tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o
> >
> > -MF
> >
> tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o.d
> >
> > -o
> >
> tools/clang/tools/extra/clangd/global-symbol-builder/CMakeFiles/global-symbol-builder.dir/GlobalSymbolBuilderMain.cpp.o
> >
> > -c
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
> > In file included from
> >
> ../tools/clang/include/clang/Frontend/CommandLineSourceLoc.h:19:0,
> > from
> > ../tools/clang/include/clang/Frontend/FrontendOptions.h:13,
> > from
> > ../tools/clang/include/clang/Frontend/CompilerInvocation.h:19,
> > from
> > ../tools/clang/include/clang/Frontend/CompilerInstance.h:16,
> > from
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:22:
> > ../include/llvm/Support/CommandLine.h:606:52: error: invalid
> cast from
> > type 'llvm::cl::opt<clang::clangd::{anonymous}::Format>' to
> type 'int'
> > llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:67:16:
> >
> > note: in expansion of macro 'clEnumValN'
> > clEnumValN(Format::YAML, "yaml",
> "human-readable YAML
> > format"),
> > ^
> > ../include/llvm/Support/CommandLine.h:606:29: error: expected
> > primary-expression before '{' token
> > llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:67:16:
> >
> > note: in expansion of macro 'clEnumValN'
> > clEnumValN(Format::YAML, "yaml",
> "human-readable YAML
> > format"),
> > ^
> > ../include/llvm/Support/CommandLine.h:606:52: error: invalid
> cast from
> > type 'llvm::cl::opt<clang::clangd::{anonymous}::Format>' to
> type 'int'
> > llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:68:16:
> >
> > note: in expansion of macro 'clEnumValN'
> > clEnumValN(Format::Binary, "binary",
> "binary RIFF
> > format")),
> > ^
> > ../include/llvm/Support/CommandLine.h:606:29: error: expected
> > primary-expression before '{' token
> > llvm::cl::OptionEnumValue { FLAGNAME, int(ENUMVAL), DESC }
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:68:16:
> >
> > note: in expansion of macro 'clEnumValN'
> > clEnumValN(Format::Binary, "binary",
> "binary RIFF
> > format")),
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:69:27:
> >
> > error: 'Format' is not a class, namespace, or enumeration
> > llvm::cl::init(Format::YAML));
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:
> >
> > In function 'int main(int, const char**)':
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:276:23:
> >
> > error: 'clang::clangd::Format' is not a class, namespace, or
> enumeration
> > case clang::clangd::Format::YAML:
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:279:23:
> >
> > error: 'clang::clangd::Format' is not a class, namespace, or
> enumeration
> > case clang::clangd::Format::Binary: {
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10:
> >
> > warning: enumeration value 'YAML' not handled in switch
> [-Wswitch]
> > switch (clang::clangd::Format) {
> > ^
> >
> ../tools/clang/tools/extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp:275:10:
> >
> > warning: enumeration value 'Binary' not handled in switch
> [-Wswitch]
> >
> > /Mikael
> >
> > > On Wed, Sep 5, 2018 at 8:46 AM Mikael Holmén
> > <mikael.holmen at ericsson.com
> <mailto:mikael.holmen at ericsson.com>
> <mailto:mikael.holmen at ericsson.com <mailto:mikael.holmen at ericsson.com>>
> > > <mailto:mikael.holmen at ericsson.com
> <mailto:mikael.holmen at ericsson.com>
> > <mailto:mikael.holmen at ericsson.com
> <mailto:mikael.holmen at ericsson.com>>>> wrote:
> > >
> > > Hi Sam,
> > >
> > > This doesn't compile for me. Both clang 3.6.0 and gcc
> 5.4.0
> > complain:
> > >
> > > [1/6] Building CXX object
> > >
> >
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o
> > > FAILED:
> > >
> >
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o
> > >
> > >
> > > /usr/bin/clang++ -march=corei7 -DGTEST_HAS_RTTI=0
> -D_DEBUG
> > > -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS
> -D__STDC_FORMAT_MACROS
> > > -D__STDC_LIMIT_MACROS -Itools/clang/tools/extra/clangd
> > > -I../tools/clang/tools/extra/clangd
> -I../tools/clang/include
> > > -Itools/clang/include -I/usr/include/libxml2 -Iinclude
> > -I../include
> > > -I/proj/flexasic/app/valgrind/3.11.0/include -fPIC
> > > -fvisibility-inlines-hidden -Werror -Werror=date-time
> > -std=c++11 -Wall
> > > -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual
> > > -Wmissing-field-initializers -pedantic -Wno-long-long
> > > -Wcovered-switch-default -Wnon-virtual-dtor
> > -Wdelete-non-virtual-dtor
> > > -Wstring-conversion -fdiagnostics-color
> -ffunction-sections
> > > -fdata-sections -fno-common -Woverloaded-virtual
> > -Wno-nested-anon-types
> > > -O3 -UNDEBUG -fno-exceptions -fno-rtti -MMD -MT
> > >
> >
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o
> > >
> > > -MF
> > >
> >
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o.d
> > >
> > > -o
> > >
> >
> tools/clang/tools/extra/clangd/CMakeFiles/clangDaemon.dir/index/Serialization.cpp.o
> > >
> > > -c
> ../tools/clang/tools/extra/clangd/index/Serialization.cpp
> > >
> ../tools/clang/tools/extra/clangd/index/Serialization.cpp:154:10:
> > > error:
> > > no viable conversion from 'clang::clangd::(anonymous
> > > namespace)::StringTableIn' to
> 'Expected<clang::clangd::(anonymous
> > > namespace)::StringTableIn>'
> > > return Table;
> > > ^~~~~
> > > ../include/llvm/Support/Error.h:434:41: note: candidate
> > constructor
> > > (the
> > > implicit copy constructor) not viable: no known
> conversion from
> > > 'clang::clangd::(anonymous namespace)::StringTableIn'
> to 'const
> > > llvm::Expected<clang::clangd::(anonymous
> > namespace)::StringTableIn> &'
> > > for 1st argument
> > > template <class T> class LLVM_NODISCARD Expected {
> > > ^
> > > ../include/llvm/Support/Error.h:456:3: note: candidate
> > constructor not
> > > viable: no known conversion from
> 'clang::clangd::(anonymous
> > > namespace)::StringTableIn' to 'llvm::Error' for 1st
> argument
> > > Expected(Error Err)
> > > ^
> > > ../include/llvm/Support/Error.h:470:3: note: candidate
> > constructor not
> > > viable: no known conversion from
> 'clang::clangd::(anonymous
> > > namespace)::StringTableIn' to 'llvm::ErrorSuccess' for 1st
> > argument
> > > Expected(ErrorSuccess) = delete;
> > > ^
> > > ../include/llvm/Support/Error.h:488:3: note: candidate
> > constructor not
> > > viable: no known conversion from
> 'clang::clangd::(anonymous
> > > namespace)::StringTableIn' to
> > 'llvm::Expected<clang::clangd::(anonymous
> > > namespace)::StringTableIn> &&' for 1st argument
> > > Expected(Expected &&Other) {
> > moveConstruct(std::move(Other)); }
> > > ^
> > > ../include/llvm/Support/Error.h:476:36: note:
> candidate template
> > > ignored: disabled by 'enable_if' [with OtherT =
> > > clang::clangd::(anonymous namespace)::StringTableIn &]
> > > typename
> std::enable_if<std::is_convertible<OtherT,
> > > T>::value>::type
> > > ^
> > > ../include/llvm/Support/Error.h:493:3: note: candidate
> template
> > > ignored:
> > > could not match 'Expected<type-parameter-0-0>' against
> > > 'clang::clangd::(anonymous namespace)::StringTableIn'
> > > Expected(Expected<OtherT> &&Other,
> > > ^
> > > In file included from
> > >
> ../tools/clang/tools/extra/clangd/index/Serialization.cpp:9:
> > > In file included from
> > >
> ../tools/clang/tools/extra/clangd/index/Serialization.h:23:
> > > In file included from
> > > ../tools/clang/tools/extra/clangd/index/Index.h:13:
> > > In file included from
> > > ../tools/clang/include/clang/Index/IndexSymbol.h:14:
> > > In file included from
> > ../tools/clang/include/clang/Lex/MacroInfo.h:18:
> > > In file included from
> > ../tools/clang/include/clang/Lex/Token.h:17:
> > > In file included from
> > > ../tools/clang/include/clang/Basic/SourceLocation.h:19:
> > > In file included from ../include/llvm/ADT/StringRef.h:13:
> > > In file included from ../include/llvm/ADT/STLExtras.h:20:
> > > ../include/llvm/ADT/Optional.h:41:28: error: call to
> > implicitly-deleted
> > > copy constructor of 'clang::clangd::SymbolSlab'
> > > new (storage.buffer) T(*O.getPointer());
> > > ^ ~~~~~~~~~~~~~~~
> > > ../include/llvm/ADT/Optional.h:141:3: note: in
> instantiation
> > of member
> > > function
> > >
> >
> 'llvm::optional_detail::OptionalStorage<clang::clangd::SymbolSlab,
> > > false>::OptionalStorage' requested here
> > > Optional(const Optional &O) = default;
> > > ^
> > >
> >
> ../tools/clang/tools/extra/clangd/index/Serialization.cpp:325:10:
> > note:
> > > in instantiation of function template specialization
> > >
> >
> 'llvm::Expected<clang::clangd::IndexFileIn>::Expected<clang::clangd::IndexFileIn
> > >
> > > &>' requested here
> > > return Result;
> > > ^
> > >
> ../tools/clang/tools/extra/clangd/index/Index.h:324:26: note:
> > copy
> > > constructor of 'SymbolSlab' is implicitly deleted because
> > field 'Arena'
> > > has a deleted copy constructor
> > > llvm::BumpPtrAllocator Arena; // Owns Symbol data
> that the
> > > Symbols do
> > > not.
> > > ^
> > > ../include/llvm/Support/Allocator.h:157:3: note: copy
> > constructor is
> > > implicitly deleted because
> > 'BumpPtrAllocatorImpl<llvm::MallocAllocator,
> > > 4096, 4096>' has a user-declared move constructor
> > > BumpPtrAllocatorImpl(BumpPtrAllocatorImpl &&Old)
> > > ^
> > > 2 errors generated.
> > >
> > > Several buildbots fail the same way.
> > >
> > > /Mikael
> > >
> > > On 09/04/2018 06:16 PM, Sam McCall via cfe-commits wrote:
> > > > Author: sammccall
> > > > Date: Tue Sep 4 09:16:50 2018
> > > > New Revision: 341375
> > > >
> > > > URL:
> http://llvm.org/viewvc/llvm-project?rev=341375&view=rev
> > > > Log:
> > > > [clangd] Define a compact binary serialization
> fomat for
> > symbol
> > > slab/index.
> > > >
> > > > Summary:
> > > > This is intended to replace the current YAML format for
> > general use.
> > > > It's ~10x more compact than YAML, and ~40% more
> compact than
> > > gzipped YAML:
> > > > llvmidx.riff = 20M, llvmidx.yaml = 272M,
> > llvmidx.yaml.gz = 32M
> > > > It's also simpler/faster to read and write.
> > > >
> > > > The format is a RIFF container (chunks of (type, size,
> > data)) with:
> > > > - a compressed string table
> > > > - simple binary encoding of symbols (with varints for
> > compactness)
> > > > It can be extended to include occurrences, Dex posting
> > lists, etc.
> > > >
> > > > There's no rich backwards-compatibility scheme, but
> a version
> > > number is included
> > > > so we can detect incompatible files and do ad-hoc
> back-compat.
> > > >
> > > > Alternatives considered:
> > > > - compressed YAML or JSON: bulky and slow to load
> > > > - llvm bitstream: confusing model and libraries
> are hard to
> > > use. My attempt
> > > > produced slightly larger files, and the code was
> > longer and
> > > slower.
> > > > - protobuf or similar: would be really nice (esp for
> > > back-compat) but the
> > > > dependency is a big hassle
> > > > - ad-hoc binary format without a container: it
> seems clear
> > > we're going
> > > > to add posting lists and occurrences here, and that
> > they will
> > > benefit
> > > > from sharing a string table. The container makes it
> > easy to debug
> > > > these pieces in isolation, and make them optional.
> > > >
> > > > Reviewers: ioeric
> > > >
> > > > Subscribers: mgorny, ilya-biryukov, MaskRay,
> jkorous, mgrang,
> > > arphaman, kadircet, cfe-commits
> > > >
> > > > Differential Revision: https://reviews.llvm.org/D51585
> > > >
> > > > Added:
> > > > clang-tools-extra/trunk/clangd/RIFF.cpp
> > > > clang-tools-extra/trunk/clangd/RIFF.h
> > > >
> clang-tools-extra/trunk/clangd/index/Serialization.cpp
> > > >
> clang-tools-extra/trunk/clangd/index/Serialization.h
> > > >
> clang-tools-extra/trunk/unittests/clangd/RIFFTests.cpp
> > > >
> > clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp
> > > > Modified:
> > > > clang-tools-extra/trunk/clangd/CMakeLists.txt
> > > >
> > >
> >
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
> > > > clang-tools-extra/trunk/clangd/index/Index.cpp
> > > > clang-tools-extra/trunk/clangd/index/Index.h
> > > >
> clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp
> > > > clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
> > > >
> clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt
> > > >
> > >
> clang-tools-extra/trunk/unittests/clangd/SymbolCollectorTests.cpp
> > > >
> > > > Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
> > > > URL:
> > >
> >
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=341375&r1=341374&r2=341375&view=diff
> > > >
> > >
> >
> ==============================================================================
> > > > --- clang-tools-extra/trunk/clangd/CMakeLists.txt
> (original)
> > > > +++ clang-tools-extra/trunk/clangd/CMakeLists.txt
> Tue Sep 4
> > > 09:16:50 2018
> > > > @@ -29,6 +29,7 @@ add_clang_library(clangDaemon
> > > > Protocol.cpp
> > > > ProtocolHandlers.cpp
> > > > Quality.cpp
> > > > + RIFF.cpp
> > > > SourceCode.cpp
> > > > Threading.cpp
> > > > Trace.cpp
> > > > @@ -41,6 +42,7 @@ add_clang_library(clangDaemon
> > > > index/Index.cpp
> > > > index/MemIndex.cpp
> > > > index/Merge.cpp
> > > > + index/Serialization.cpp
> > > > index/SymbolCollector.cpp
> > > > index/SymbolYAML.cpp
> > > >
> > > >
> > > > Added: clang-tools-extra/trunk/clangd/RIFF.cpp
> > > > URL:
> > >
> >
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.cpp?rev=341375&view=auto
> > > >
> > >
> >
> ==============================================================================
> > > > --- clang-tools-extra/trunk/clangd/RIFF.cpp (added)
> > > > +++ clang-tools-extra/trunk/clangd/RIFF.cpp Tue Sep 4
> > 09:16:50 2018
> > > > @@ -0,0 +1,88 @@
> > > > +//===--- RIFF.cpp - Binary container file format
> > > --------------------------===//
> > > > +//
> > > > +// The LLVM Compiler
> Infrastructure
> > > > +//
> > > > +// This file is distributed under the University
> of Illinois
> > > Open Source
> > > > +// License. See LICENSE.TXT for details.
> > > > +//
> > > >
> > >
> >
> +//===----------------------------------------------------------------------===//
> > > > +
> > > > +#include "RIFF.h"
> > > > +#include "llvm/Support/Endian.h"
> > > > +
> > > > +using namespace llvm;
> > > > +namespace clang {
> > > > +namespace clangd {
> > > > +namespace riff {
> > > > +
> > > > +static Error makeError(const char *Msg) {
> > > > + return
> createStringError(inconvertibleErrorCode(), Msg);
> > > > +}
> > > > +
> > > > +Expected<Chunk> readChunk(StringRef &Stream) {
> > > > + if (Stream.size() < 8)
> > > > + return makeError("incomplete chunk header");
> > > > + Chunk C;
> > > > + std::copy(Stream.begin(), Stream.begin() + 4,
> > C.ID.begin());
> > > > + Stream = Stream.drop_front(4);
> > > > + uint32_t Len =
> > > support::endian::read32le(Stream.take_front(4).begin());
> > > > + Stream = Stream.drop_front(4);
> > > > + if (Stream.size() < Len)
> > > > + return makeError("truncated chunk");
> > > > + C.Data = Stream.take_front(Len);
> > > > + Stream = Stream.drop_front(Len);
> > > > + if (Len % 2 & !Stream.empty()) { // Skip padding
> byte.
> > > > + if (Stream.front())
> > > > + return makeError("nonzero padding byte");
> > > > + Stream = Stream.drop_front();
> > > > + }
> > > > + return C;
> > > > +};
> > > > +
> > > > +raw_ostream &operator<<(raw_ostream &OS, const
> Chunk &C) {
> > > > + OS.write(C.ID.begin(), C.ID.size());
> > > > + char Size[4];
> > > > + llvm::support::endian::write32le(Size,
> C.Data.size());
> > > > + OS.write(Size, sizeof(Size));
> > > > + OS << C.Data;
> > > > + if (C.Data.size() % 2)
> > > > + OS.write(0);
> > > > + return OS;
> > > > +}
> > > > +
> > > > +llvm::Expected<File> readFile(llvm::StringRef
> Stream) {
> > > > + auto RIFF = readChunk(Stream);
> > > > + if (!RIFF)
> > > > + return RIFF.takeError();
> > > > + if (RIFF->ID != fourCC("RIFF"))
> > > > + return makeError("not a RIFF container");
> > > > + if (RIFF->Data.size() < 4)
> > > > + return makeError("RIFF chunk too short");
> > > > + File F;
> > > > + std::copy(RIFF->Data.begin(), RIFF->Data.begin()
> + 4,
> > > F.Type.begin());
> > > > + for (llvm::StringRef Body =
> RIFF->Data.drop_front(4);
> > > !Body.empty();)
> > > > + if (auto Chunk = readChunk(Body)) {
> > > > + F.Chunks.push_back(*Chunk);
> > > > + } else
> > > > + return Chunk.takeError();
> > > > + return F;
> > > > +}
> > > > +
> > > > +raw_ostream &operator<<(raw_ostream &OS, const
> File &F) {
> > > > + // To avoid copies, we serialize the outer RIFF
> chunk
> > "by hand".
> > > > + size_t DataLen = 4; // Predict length of RIFF
> chunk data.
> > > > + for (const auto &C : F.Chunks)
> > > > + DataLen += 4 + 4 + C.Data.size() +
> (C.Data.size() % 2);
> > > > + OS << "RIFF";
> > > > + char Size[4];
> > > > + llvm::support::endian::write32le(Size, DataLen);
> > > > + OS.write(Size, sizeof(Size));
> > > > + OS.write(F.Type.begin(), F.Type.size());
> > > > + for (const auto &C : F.Chunks)
> > > > + OS << C;
> > > > + return OS;
> > > > +}
> > > > +
> > > > +} // namespace riff
> > > > +} // namespace clangd
> > > > +} // namespace clang
> > > >
> > > > Added: clang-tools-extra/trunk/clangd/RIFF.h
> > > > URL:
> > >
> >
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/RIFF.h?rev=341375&view=auto
> > > >
> > >
> >
> ==============================================================================
> > > > --- clang-tools-extra/trunk/clangd/RIFF.h (added)
> > > > +++ clang-tools-extra/trunk/clangd/RIFF.h Tue Sep 4
> > 09:16:50 2018
> > > > @@ -0,0 +1,81 @@
> > > > +//===--- RIFF.h - Binary container file format
> > > -------------------*- C++-*-===//
> > > > +//
> > > > +// The LLVM Compiler
> Infrastructure
> > > > +//
> > > > +// This file is distributed under the University
> of Illinois
> > > Open Source
> > > > +// License. See LICENSE.TXT for details.
> > > > +//
> > > >
> > >
> >
> +//===----------------------------------------------------------------------===//
> > > > +//
> > > > +// Tools for reading and writing data in RIFF
> containers.
> > > > +//
> > > > +// A chunk consists of:
> > > > +// - ID : char[4]
> > > > +// - Length : uint32
> > > > +// - Data : byte[Length]
> > > > +// - Padding : byte[Length % 2]
> > > > +// The semantics of a chunk's Data are determined
> by its ID.
> > > > +// The format makes it easy to skip over
> uninteresting or
> > > unknown chunks.
> > > > +//
> > > > +// A RIFF file is a single chunk with ID "RIFF".
> Its Data is:
> > > > +// - Type : char[4]
> > > > +// - Chunks : chunk[]
> > > > +//
> > > > +// This means that a RIFF file consists of:
> > > > +// - "RIFF" : char[4]
> > > > +// - File length - 8 : uint32
> > > > +// - File type : char[4]
> > > > +// - Chunks : chunk[]
> > > > +//
> > > >
> > >
> >
> +//===----------------------------------------------------------------------===//
> > > > +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_RIFF_H
> > > > +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_RIFF_H
> > > > +#include "llvm/ADT/StringRef.h"
> > > > +#include "llvm/Support/Error.h"
> > > > +#include "llvm/Support/ScopedPrinter.h"
> > > > +#include <array>
> > > > +
> > > > +namespace clang {
> > > > +namespace clangd {
> > > > +namespace riff {
> > > > +
> > > > +// A FourCC identifies a chunk in a file, or the
> type of file
> > > itself.
> > > > +using FourCC = std::array<char, 4>;
> > > > +// Get a FourCC from a string literal, e.g.
> fourCC("RIFF").
> > > > +inline constexpr FourCC fourCC(const char
> (&Literal)[5]) {
> > > > + return FourCC{{Literal[0], Literal[1], Literal[2],
> > Literal[3]}};
> > > > +}
> > > > +// A chunk is a section in a RIFF container.
> > > > +struct Chunk {
> > > > + FourCC ID;
> > > > + llvm::StringRef Data;
> > > > +};
> > > > +inline bool operator==(const Chunk &L, const Chunk
> &R) {
> > > > + return std::tie(L.ID <http://L.ID> <http://L.ID>
> <http://L.ID>,
> > L.Data) == std::tie(R.ID <http://R.ID> <http://R.ID>
> > > <http://R.ID>, R.Data);
> > > > +}
> > > > +// A File is a RIFF container, which is a typed chunk
> > sequence.
> > > > +struct File {
> > > > + FourCC Type;
> > > > + std::vector<Chunk> Chunks;
> > > > +};
> > > > +inline bool operator==(const File &L, const File &R) {
> > > > + return std::tie(L.Type, L.Chunks) ==
> std::tie(R.Type,
> > R.Chunks);
> > > > +}
> > > > +
> > > > +// Reads a single chunk from the start of Stream.
> > > > +// Stream is updated to exclude the consumed chunk.
> > > > +llvm::Expected<Chunk> readChunk(llvm::StringRef
> &Stream);
> > > > +
> > > > +// Serialize a single chunk to OS.
> > > > +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
> > const Chunk &);
> > > > +
> > > > +// Parses a RIFF file consisting of a single RIFF
> chunk.
> > > > +llvm::Expected<File> readFile(llvm::StringRef Stream);
> > > > +
> > > > +// Serialize a RIFF file (i.e. a single RIFF
> chunk) to OS.
> > > > +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
> > const File &);
> > > > +
> > > > +} // namespace riff
> > > > +} // namespace clangd
> > > > +} // namespace clang
> > > > +#endif
> > > >
> > > > Modified:
> > >
> >
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
> > > > URL:
> > >
> >
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp?rev=341375&r1=341374&r2=341375&view=diff
> > > >
> > >
> >
> ==============================================================================
> > > > ---
> > >
> >
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
> > > (original)
> > > > +++
> > >
> >
> clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
> > > Tue Sep 4 09:16:50 2018
> > > > @@ -7,15 +7,16 @@
> > > > //
> > > >
> > >
> >
> //===----------------------------------------------------------------------===//
> > > > //
> > > > -// GlobalSymbolBuilder is a tool to generate
> YAML-format
> > symbols
> > > across the
> > > > -// whole project. This tools is for **experimental**
> > only. Don't
> > > use it in
> > > > -// production code.
> > > > +// GlobalSymbolBuilder is a tool to extract
> symbols from
> > a whole
> > > project.
> > > > +// This tool is **experimental** only. Don't use it in
> > > production code.
> > > > //
> > > >
> > >
> >
> //===----------------------------------------------------------------------===//
> > > >
> > > > +#include "RIFF.h"
> > > > #include "index/CanonicalIncludes.h"
> > > > #include "index/Index.h"
> > > > #include "index/Merge.h"
> > > > +#include "index/Serialization.h"
> > > > #include "index/SymbolCollector.h"
> > > > #include "index/SymbolYAML.h"
> > > > #include "clang/Frontend/CompilerInstance.h"
> > > > @@ -59,6 +60,14 @@ static llvm::cl::opt<bool>
> MergeOnTheFly
> > > > "MapReduce."),
> > > > llvm::cl::init(true), llvm::cl::Hidden);
> > > >
> > > > +enum class Format { YAML, Binary };
> > > > +static llvm::cl::opt<Format>
> > > > + Format("format", llvm::cl::desc("Format of the
> index
> > to be
> > > written"),
> > > > + llvm::cl::values(
> > > > + clEnumValN(Format::YAML, "yaml",
> > "human-readable
> > > YAML format"),
> > > > + clEnumValN(Format::Binary, "binary",
> > "binary RIFF
> > > format")),
> > > > + llvm::cl::init(Format::YAML));
> > > > +
> > > > /// Responsible for aggregating symbols from each
> processed
> > > file and producing
> > > > /// the final results. All methods in this class
> must be
> > > thread-safe,
> > > > /// 'consumeSymbols' may be called from multiple
> threads.
> > > > @@ -210,8 +219,8 @@ int main(int argc, const char
> **argv) {
> > > > llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
> > > >
> > > > const char *Overview = R"(
> > > > - This is an **experimental** tool to generate
> YAML-format
> > > project-wide symbols
> > > > - for clangd (global code completion). It would be
> > changed and
> > > deprecated
> > > > + This is an **experimental** tool to extract
> symbols from a
> > > whole project
> > > > + for clangd (global code completion). It will be
> changed and
> > > deprecated
> > > > eventually. Don't use it in production code!
> > > >
> > > > Example usage for building index for the whole
> project
> > using
> > > CMake compile
> > > > @@ -262,7 +271,16 @@ int main(int argc, const char
> **argv) {
> > > > }
> > > > // Reduce phase: combine symbols with the same IDs.
> > > > auto UniqueSymbols = Consumer->mergeResults();
> > > > - // Output phase: emit YAML for result symbols.
> > > > - SymbolsToYAML(UniqueSymbols, llvm::outs());
> > > > + // Output phase: emit result symbols.
> > > > + switch (clang::clangd::Format) {
> > > > + case clang::clangd::Format::YAML:
> > > > + SymbolsToYAML(UniqueSymbols, llvm::outs());
> > > > + break;
> > > > + case clang::clangd::Format::Binary: {
> > > > + clang::clangd::IndexFileOut Out;
> > > > + Out.Symbols = &UniqueSymbols;
> > > > + llvm::outs() << Out;
> > > > + }
> > > > + }
> > > > return 0;
> > > > }
> > > >
> > > > Modified:
> clang-tools-extra/trunk/clangd/index/Index.cpp
> > > > URL:
> > >
> >
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341375&r1=341374&r2=341375&view=diff
> > > >
> > >
> >
> ==============================================================================
> > > > --- clang-tools-extra/trunk/clangd/index/Index.cpp
> (original)
> > > > +++ clang-tools-extra/trunk/clangd/index/Index.cpp
> Tue Sep 4
> > > 09:16:50 2018
> > > > @@ -10,6 +10,7 @@
> > > > #include "Index.h"
> > > > #include "llvm/ADT/StringExtras.h"
> > > > #include "llvm/ADT/StringRef.h"
> > > > +#include "llvm/Support/Error.h"
> > > > #include "llvm/Support/SHA1.h"
> > > > #include "llvm/Support/raw_ostream.h"
> > > >
> > > > @@ -28,21 +29,20 @@ SymbolID::SymbolID(StringRef USR)
> > > > :
> HashValue(SHA1::hash(arrayRefFromStringRef(USR))) {}
> > > >
> > > > raw_ostream &operator<<(raw_ostream &OS, const
> SymbolID
> > &ID) {
> > > > - OS << toHex(toStringRef(ID.HashValue));
> > > > - return OS;
> > > > + return OS << toHex(ID.raw());
> > > > }
> > > >
> > > > -std::string SymbolID::str() const {
> > > > - std::string ID;
> > > > - llvm::raw_string_ostream OS(ID);
> > > > - OS << *this;
> > > > - return OS.str();
> > > > +SymbolID SymbolID::fromRaw(llvm::StringRef Raw) {
> > > > + SymbolID ID;
> > > > + assert(Raw.size() == RawSize);
> > > > + memcpy(ID.HashValue.data(), Raw.data(), RawSize);
> > > > + return ID;
> > > > }
> > > >
> > > > +std::string SymbolID::str() const { return
> toHex(raw()); }
> > > > +
> > > > void operator>>(StringRef Str, SymbolID &ID) {
> > > > - std::string HexString = fromHex(Str);
> > > > - assert(HexString.size() == ID.HashValue.size());
> > > > - std::copy(HexString.begin(), HexString.end(),
> > > ID.HashValue.begin());
> > > > + ID = SymbolID::fromRaw(fromHex(Str));
> > > > }
> > > >
> > > > raw_ostream &operator<<(raw_ostream &OS,
> SymbolOrigin O) {
> > > > @@ -78,34 +78,18 @@ SymbolSlab::const_iterator
> SymbolSlab::f
> > > > }
> > > >
> > > > // Copy the underlying data of the symbol into
> the owned
> > arena.
> > > > -static void own(Symbol &S, llvm::UniqueStringSaver
> &Strings,
> > > > - BumpPtrAllocator &Arena) {
> > > > - // Intern replaces V with a reference to the same
> > string owned
> > > by the arena.
> > > > - auto Intern = [&](StringRef &V) { V =
> Strings.save(V); };
> > > > -
> > > > - // We need to copy every StringRef field onto
> the arena.
> > > > - Intern(S.Name);
> > > > - Intern(S.Scope);
> > > > - Intern(S.CanonicalDeclaration.FileURI);
> > > > - Intern(S.Definition.FileURI);
> > > > -
> > > > - Intern(S.Signature);
> > > > - Intern(S.CompletionSnippetSuffix);
> > > > -
> > > > - Intern(S.Documentation);
> > > > - Intern(S.ReturnType);
> > > > - for (auto &I : S.IncludeHeaders)
> > > > - Intern(I.IncludeHeader);
> > > > +static void own(Symbol &S, llvm::UniqueStringSaver
> > &Strings) {
> > > > + visitStrings(S, [&](StringRef &V) { V =
> > Strings.save(V); });
> > > > }
> > > >
> > > > void SymbolSlab::Builder::insert(const Symbol &S) {
> > > > auto R = SymbolIndex.try_emplace(S.ID
> <http://S.ID> <http://S.ID>
> > <http://S.ID>,
> > > Symbols.size());
> > > > if (R.second) {
> > > > Symbols.push_back(S);
> > > > - own(Symbols.back(), UniqueStrings, Arena);
> > > > + own(Symbols.back(), UniqueStrings);
> > > > } else {
> > > > auto &Copy = Symbols[R.first->second] = S;
> > > > - own(Copy, UniqueStrings, Arena);
> > > > + own(Copy, UniqueStrings);
> > > > }
> > > > }
> > > >
> > > > @@ -118,7 +102,7 @@ SymbolSlab
> SymbolSlab::Builder::build()
> > > > BumpPtrAllocator NewArena;
> > > > llvm::UniqueStringSaver Strings(NewArena);
> > > > for (auto &S : Symbols)
> > > > - own(S, Strings, NewArena);
> > > > + own(S, Strings);
> > > > return SymbolSlab(std::move(NewArena),
> > std::move(Symbols));
> > > > }
> > > >
> > > >
> > > > Modified: clang-tools-extra/trunk/clangd/index/Index.h
> > > > URL:
> > >
> >
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=341375&r1=341374&r2=341375&view=diff
> > > >
> > >
> >
> ==============================================================================
> > > > --- clang-tools-extra/trunk/clangd/index/Index.h
> (original)
> > > > +++ clang-tools-extra/trunk/clangd/index/Index.h
> Tue Sep 4
> > > 09:16:50 2018
> > > > @@ -84,26 +84,28 @@ public:
> > > > return HashValue < Sym.HashValue;
> > > > }
> > > >
> > > > + constexpr static size_t RawSize = 20;
> > > > + llvm::StringRef raw() const {
> > > > + return StringRef(reinterpret_cast<const char
> > > *>(HashValue.data()), RawSize);
> > > > + }
> > > > + static SymbolID fromRaw(llvm::StringRef);
> > > > // Returns a 40-bytes hex encoded string.
> > > > std::string str() const;
> > > >
> > > > private:
> > > > - static constexpr unsigned HashByteLength = 20;
> > > > -
> > > > - friend llvm::hash_code hash_value(const SymbolID
> &ID) {
> > > > - // We already have a good hash, just return
> the first
> > bytes.
> > > > - static_assert(sizeof(size_t) <=
> HashByteLength, "size_t
> > > longer than SHA1!");
> > > > - size_t Result;
> > > > - memcpy(&Result, ID.HashValue.data(),
> sizeof(size_t));
> > > > - return llvm::hash_code(Result);
> > > > - }
> > > > - friend llvm::raw_ostream
> &operator<<(llvm::raw_ostream &OS,
> > > > - const
> SymbolID &ID);
> > > > friend void operator>>(llvm::StringRef Str,
> SymbolID &ID);
> > > >
> > > > - std::array<uint8_t, HashByteLength> HashValue;
> > > > + std::array<uint8_t, RawSize> HashValue;
> > > > };
> > > >
> > > > +inline llvm::hash_code hash_value(const SymbolID
> &ID) {
> > > > + // We already have a good hash, just return the
> first
> > bytes.
> > > > + assert(sizeof(size_t) <= SymbolID::RawSize &&
> "size_t
> > longer
> > > than SHA1!");
> > > > + size_t Result;
> > > > + memcpy(&Result, ID.raw().data(), sizeof(size_t));
> > > > + return llvm::hash_code(Result);
> > > > +}
> > > > +
> > > > // Write SymbolID into the given stream. SymbolID is
> > encoded as
> > > a 40-bytes
> > > > // hex string.
> > > > llvm::raw_ostream &operator<<(llvm::raw_ostream
> &OS, const
> > > SymbolID &ID);
> > > > @@ -246,6 +248,21 @@ struct Symbol {
> > > > };
> > > > llvm::raw_ostream &operator<<(llvm::raw_ostream
> &OS, const
> > > Symbol &S);
> > > >
> > > > +// Invokes Callback with each StringRef& contained
> in the
> > Symbol.
> > > > +// Useful for deduplicating backing strings.
> > > > +template <typename Callback> void
> visitStrings(Symbol &S,
> > const
> > > Callback &CB) {
> > > > + CB(S.Name);
> > > > + CB(S.Scope);
> > > > + CB(S.CanonicalDeclaration.FileURI);
> > > > + CB(S.Definition.FileURI);
> > > > + CB(S.Signature);
> > > > + CB(S.CompletionSnippetSuffix);
> > > > + CB(S.Documentation);
> > > > + CB(S.ReturnType);
> > > > + for (auto &Include : S.IncludeHeaders)
> > > > + CB(Include.IncludeHeader);
> > > > +}
> > > > +
> > > > // Computes query-independent quality score for a
> Symbol.
> > > > // This currently falls in the range [1, ln(#indexed
> > documents)].
> > > > // FIXME: this should probably be split into
> symbol ->
> > signals
> > > >
> > > > Added:
> clang-tools-extra/trunk/clangd/index/Serialization.cpp
> > > > URL:
> > >
> >
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=341375&view=auto
> > > >
> > >
> >
> ==============================================================================
> > > > ---
> clang-tools-extra/trunk/clangd/index/Serialization.cpp
> > (added)
> > > > +++
> clang-tools-extra/trunk/clangd/index/Serialization.cpp Tue
> > > Sep 4 09:16:50 2018
> > > > @@ -0,0 +1,366 @@
> > > > +//===-- Serialization.cpp - Binary serialization
> of index
> > data
> > > ------------===//
> > > > +//
> > > > +// The LLVM Compiler
> Infrastructure
> > > > +//
> > > > +// This file is distributed under the University
> of Illinois
> > > Open Source
> > > > +// License. See LICENSE.TXT for details.
> > > > +//
> > > >
> > >
> >
> +//===----------------------------------------------------------------------===//
> > > > +#include "Serialization.h"
> > > > +#include "../RIFF.h"
> > > > +#include "llvm/Support/Compression.h"
> > > > +#include "llvm/Support/Endian.h"
> > > > +#include "llvm/Support/Error.h"
> > > > +
> > > > +using namespace llvm;
> > > > +namespace clang {
> > > > +namespace clangd {
> > > > +namespace {
> > > > +Error makeError(const Twine &Msg) {
> > > > + return make_error<StringError>(Msg,
> > inconvertibleErrorCode());
> > > > +}
> > > > +
> > > > +// IO PRIMITIVES
> > > > +// We use little-endian 32 bit ints, sometimes with
> > > variable-length encoding.
> > > > +
> > > > +StringRef consume(StringRef &Data, int N) {
> > > > + StringRef Ret = Data.take_front(N);
> > > > + Data = Data.drop_front(N);
> > > > + return Ret;
> > > > +}
> > > > +
> > > > +uint8_t consume8(StringRef &Data) {
> > > > + uint8_t Ret = Data.front();
> > > > + Data = Data.drop_front();
> > > > + return Ret;
> > > > +}
> > > > +
> > > > +uint32_t consume32(StringRef &Data) {
> > > > + auto Ret =
> support::endian::read32le(Data.bytes_begin());
> > > > + Data = Data.drop_front(4);
> > > > + return Ret;
> > > > +}
> > > > +
> > > > +void write32(uint32_t I, raw_ostream &OS) {
> > > > + char buf[4];
> > > > + support::endian::write32le(buf, I);
> > > > + OS.write(buf, sizeof(buf));
> > > > +}
> > > > +
> > > > +// Variable-length int encoding (varint) uses the
> bottom
> > 7 bits
> > > of each byte
> > > > +// to encode the number, and the top bit to
> indicate whether
> > > more bytes follow.
> > > > +// e.g. 9a 2f means [0x1a and keep reading, 0x2f
> and stop].
> > > > +// This represents 0x1a | 0x2f<<7 = 6042.
> > > > +// A 32-bit integer takes 1-5 bytes to encode; small
> > numbers are
> > > more compact.
> > > > +void writeVar(uint32_t I, raw_ostream &OS) {
> > > > + constexpr static uint8_t More = 1 << 7;
> > > > + if (LLVM_LIKELY(I < 1 << 7)) {
> > > > + OS.write(I);
> > > > + return;
> > > > + }
> > > > + for (;;) {
> > > > + OS.write(I | More);
> > > > + I >>= 7;
> > > > + if (I < 1 << 7) {
> > > > + OS.write(I);
> > > > + return;
> > > > + }
> > > > + }
> > > > +}
> > > > +
> > > > +uint32_t consumeVar(StringRef &Data) {
> > > > + constexpr static uint8_t More = 1 << 7;
> > > > + uint8_t B = consume8(Data);
> > >
>
More information about the cfe-commits
mailing list