[lld] r286061 - [ELF] - Implemented threaded --build-id computation
Rui Ueyama via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 15 13:25:53 PST 2016
On Tue, Nov 15, 2016 at 11:32 AM, Rafael Espíndola <
rafael.espindola at gmail.com> wrote:
> Have you benchmarked this on windows? I was about to benchmark this on
> linux but noticed that parallel_for_each does nothing there since it
> uses a thread for each 1024 elements, so in this case it would only
> use more than one thread if the file was more than a GiB.
>
> Any idea why parallel_for_each is implemented that way? It would seem
> more natural to create a task for each element and let the caller
> create more coarse elements if desired.
>
I think you can change the code. Our task is fairly coarse, like computing
a cryptic hash for a 1 MB blob, or copy & apply relocations for a section,
so one vector element can be one task unit.
Cheers,
> Rafael
>
>
> On 6 November 2016 at 02:42, George Rimar via llvm-commits
> <llvm-commits at lists.llvm.org> wrote:
> > Author: grimar
> > Date: Sun Nov 6 01:42:55 2016
> > New Revision: 286061
> >
> > URL: http://llvm.org/viewvc/llvm-project?rev=286061&view=rev
> > Log:
> > [ELF] - Implemented threaded --build-id computation
> >
> > Patch switches computing of --build-id hash to tree.
> >
> > This is the way when input data is splitted by chunks,
> > hash is computed for each one in threaded/non-threaded way.
> > At the end hash is conputed for result tree.
> >
> > With or without -threads the result hash is the same.
> >
> > Differential revision: https://reviews.llvm.org/D26199
> >
> > Modified:
> > lld/trunk/ELF/SyntheticSections.cpp
> > lld/trunk/ELF/SyntheticSections.h
> > lld/trunk/test/ELF/build-id.s
> >
> > Modified: lld/trunk/ELF/SyntheticSections.cpp
> > URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/
> SyntheticSections.cpp?rev=286061&r1=286060&r2=286061&view=diff
> > ============================================================
> ==================
> > --- lld/trunk/ELF/SyntheticSections.cpp (original)
> > +++ lld/trunk/ELF/SyntheticSections.cpp Sun Nov 6 01:42:55 2016
> > @@ -23,6 +23,7 @@
> > #include "Strings.h"
> > #include "SymbolTable.h"
> >
> > +#include "lld/Core/Parallel.h"
> > #include "llvm/Support/Endian.h"
> > #include "llvm/Support/MD5.h"
> > #include "llvm/Support/RandomNumberGenerator.h"
> > @@ -91,7 +92,8 @@ InterpSection<ELFT>::InterpSection()
> > template <class ELFT>
> > BuildIdSection<ELFT>::BuildIdSection(size_t HashSize)
> > : InputSection<ELFT>(SHF_ALLOC, SHT_NOTE, 1, ArrayRef<uint8_t>(),
> > - ".note.gnu.build-id") {
> > + ".note.gnu.build-id"),
> > + HashSize(HashSize) {
> > this->Live = true;
> >
> > Buf.resize(16 + HashSize);
> > @@ -108,29 +110,66 @@ uint8_t *BuildIdSection<ELFT>::getOutput
> > return Start + this->OutSec->getFileOffset() + this->OutSecOff;
> > }
> >
> > +static std::vector<ArrayRef<uint8_t>> split(ArrayRef<uint8_t> Arr,
> > + size_t ChunkSize) {
> > + std::vector<ArrayRef<uint8_t>> Ret;
> > + while (Arr.size() > ChunkSize) {
> > + Ret.push_back(Arr.take_front(ChunkSize));
> > + Arr = Arr.drop_front(ChunkSize);
> > + }
> > + if (!Arr.empty())
> > + Ret.push_back(Arr);
> > + return Ret;
> > +}
> > +
> > template <class ELFT>
> > -void BuildIdFastHash<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf)
> {
> > - const endianness E = ELFT::TargetEndianness;
> > +void BuildIdSection<ELFT>::computeHash(
> > + llvm::ArrayRef<uint8_t> Data,
> > + std::function<void(ArrayRef<uint8_t> Arr, uint8_t *Hash)> Hash) {
> > + std::vector<ArrayRef<uint8_t>> Chunks = split(Data, 1024 * 1024);
> > + std::vector<uint8_t> HashList(Chunks.size() * HashSize);
> > +
> > + if (Config->Threads)
> > + parallel_for_each(Chunks.begin(), Chunks.end(),
> > + [&](ArrayRef<uint8_t> &Chunk) {
> > + size_t Id = &Chunk - Chunks.data();
> > + Hash(Chunk, HashList.data() + Id * HashSize);
> > + });
> > + else
> > + std::for_each(Chunks.begin(), Chunks.end(), [&](ArrayRef<uint8_t>
> &Chunk) {
> > + size_t Id = &Chunk - Chunks.data();
> > + Hash(Chunk, HashList.data() + Id * HashSize);
> > + });
> >
> > - // 64-bit xxhash
> > - uint64_t Hash = xxHash64(toStringRef(Buf));
> > - write64<E>(this->getOutputLoc(Buf.begin()) + 16, Hash);
> > + Hash(HashList, this->getOutputLoc((uint8_t *)Data.begin()) + 16);
> > +}
> > +
> > +template <class ELFT>
> > +void BuildIdFastHash<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf)
> {
> > + computeHash(Buf, [](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
> > + uint64_t Hash = xxHash64(toStringRef(Arr));
> > + write64<ELFT::TargetEndianness>(Dest, Hash);
> > + });
> > }
> >
> > template <class ELFT>
> > void BuildIdMd5<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
> > - MD5 Hash;
> > - Hash.update(Buf);
> > - MD5::MD5Result Res;
> > - Hash.final(Res);
> > - memcpy(this->getOutputLoc(Buf.begin()) + 16, Res, 16);
> > + computeHash(Buf, [&](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
> > + MD5 Hash;
> > + Hash.update(Arr);
> > + MD5::MD5Result Res;
> > + Hash.final(Res);
> > + memcpy(Dest, Res, HashSize);
> > + });
> > }
> >
> > template <class ELFT>
> > void BuildIdSha1<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
> > - SHA1 Hash;
> > - Hash.update(Buf);
> > - memcpy(this->getOutputLoc(Buf.begin()) + 16, Hash.final().data(),
> 20);
> > + computeHash(Buf, [&](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
> > + SHA1 Hash;
> > + Hash.update(Arr);
> > + memcpy(Dest, Hash.final().data(), HashSize);
> > + });
> > }
> >
> > template <class ELFT>
> >
> > Modified: lld/trunk/ELF/SyntheticSections.h
> > URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/
> SyntheticSections.h?rev=286061&r1=286060&r2=286061&view=diff
> > ============================================================
> ==================
> > --- lld/trunk/ELF/SyntheticSections.h (original)
> > +++ lld/trunk/ELF/SyntheticSections.h Sun Nov 6 01:42:55 2016
> > @@ -38,6 +38,12 @@ public:
> > protected:
> > BuildIdSection(size_t HashSize);
> > std::vector<uint8_t> Buf;
> > +
> > + void
> > + computeHash(llvm::ArrayRef<uint8_t> Buf,
> > + std::function<void(ArrayRef<uint8_t> Arr, uint8_t
> *Hash)> Hash);
> > +
> > + size_t HashSize;
> > };
> >
> > template <class ELFT>
> >
> > Modified: lld/trunk/test/ELF/build-id.s
> > URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/
> build-id.s?rev=286061&r1=286060&r2=286061&view=diff
> > ============================================================
> ==================
> > --- lld/trunk/test/ELF/build-id.s (original)
> > +++ lld/trunk/test/ELF/build-id.s Sun Nov 6 01:42:55 2016
> > @@ -16,6 +16,14 @@
> > # RUN: ld.lld --build-id=md5 --build-id=none %t -o %t2
> > # RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=NONE %s
> >
> > +## Multithreaded cases:
> > +# RUN: ld.lld --build-id -threads %t -o %t2
> > +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=DEFAULT %s
> > +# RUN: ld.lld --build-id=md5 -threads %t -o %t2
> > +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=MD5 %s
> > +# RUN: ld.lld --build-id=sha1 -threads %t -o %t2
> > +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=SHA1 %s
> > +
> > .globl _start
> > _start:
> > nop
> > @@ -26,12 +34,19 @@ _start:
> > # DEFAULT: Contents of section .note.test:
> > # DEFAULT: Contents of section .note.gnu.build-id:
> > # DEFAULT-NEXT: 04000000 08000000 03000000 474e5500 ............GNU.
> > +# DEFAULT-NEXT: ab
> >
> > # MD5: Contents of section .note.gnu.build-id:
> > # MD5-NEXT: 04000000 10000000 03000000 474e5500 ............GNU.
> > +# MD5-NEXT: 29
> >
> > # SHA1: Contents of section .note.gnu.build-id:
> > # SHA1-NEXT: 04000000 14000000 03000000 474e5500 ............GNU.
> > +# SHA1-NEXT: b1
> > +
> > +# TREE: Contents of section .note.gnu.build-id:
> > +# TREE-NEXT: 04000000 14000000 03000000 474e5500 ............GNU.
> > +# TREE-NEXT: 18
> >
> > # UUID: Contents of section .note.gnu.build-id:
> > # UUID-NEXT: 04000000 10000000 03000000 474e5500 ............GNU.
> >
> >
> > _______________________________________________
> > llvm-commits mailing list
> > llvm-commits at lists.llvm.org
> > http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20161115/ebdbb73d/attachment.html>
More information about the llvm-commits
mailing list