[lld] r286061 - [ELF] - Implemented threaded --build-id computation
Rafael Espíndola via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 15 11:32:07 PST 2016
Have you benchmarked this on windows? I was about to benchmark this on
linux but noticed that parallel_for_each does nothing there since it
uses a thread for each 1024 elements, so in this case it would only
use more than one thread if the file was more than a GiB.
Any idea why parallel_for_each is implemented that way? It would seem
more natural to create a task for each element and let the caller
create more coarse elements if desired.
Cheers,
Rafael
On 6 November 2016 at 02:42, George Rimar via llvm-commits
<llvm-commits at lists.llvm.org> wrote:
> Author: grimar
> Date: Sun Nov 6 01:42:55 2016
> New Revision: 286061
>
> URL: http://llvm.org/viewvc/llvm-project?rev=286061&view=rev
> Log:
> [ELF] - Implemented threaded --build-id computation
>
> Patch switches computing of --build-id hash to tree.
>
> This is the way when input data is splitted by chunks,
> hash is computed for each one in threaded/non-threaded way.
> At the end hash is conputed for result tree.
>
> With or without -threads the result hash is the same.
>
> Differential revision: https://reviews.llvm.org/D26199
>
> Modified:
> lld/trunk/ELF/SyntheticSections.cpp
> lld/trunk/ELF/SyntheticSections.h
> lld/trunk/test/ELF/build-id.s
>
> Modified: lld/trunk/ELF/SyntheticSections.cpp
> URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/SyntheticSections.cpp?rev=286061&r1=286060&r2=286061&view=diff
> ==============================================================================
> --- lld/trunk/ELF/SyntheticSections.cpp (original)
> +++ lld/trunk/ELF/SyntheticSections.cpp Sun Nov 6 01:42:55 2016
> @@ -23,6 +23,7 @@
> #include "Strings.h"
> #include "SymbolTable.h"
>
> +#include "lld/Core/Parallel.h"
> #include "llvm/Support/Endian.h"
> #include "llvm/Support/MD5.h"
> #include "llvm/Support/RandomNumberGenerator.h"
> @@ -91,7 +92,8 @@ InterpSection<ELFT>::InterpSection()
> template <class ELFT>
> BuildIdSection<ELFT>::BuildIdSection(size_t HashSize)
> : InputSection<ELFT>(SHF_ALLOC, SHT_NOTE, 1, ArrayRef<uint8_t>(),
> - ".note.gnu.build-id") {
> + ".note.gnu.build-id"),
> + HashSize(HashSize) {
> this->Live = true;
>
> Buf.resize(16 + HashSize);
> @@ -108,29 +110,66 @@ uint8_t *BuildIdSection<ELFT>::getOutput
> return Start + this->OutSec->getFileOffset() + this->OutSecOff;
> }
>
> +static std::vector<ArrayRef<uint8_t>> split(ArrayRef<uint8_t> Arr,
> + size_t ChunkSize) {
> + std::vector<ArrayRef<uint8_t>> Ret;
> + while (Arr.size() > ChunkSize) {
> + Ret.push_back(Arr.take_front(ChunkSize));
> + Arr = Arr.drop_front(ChunkSize);
> + }
> + if (!Arr.empty())
> + Ret.push_back(Arr);
> + return Ret;
> +}
> +
> template <class ELFT>
> -void BuildIdFastHash<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
> - const endianness E = ELFT::TargetEndianness;
> +void BuildIdSection<ELFT>::computeHash(
> + llvm::ArrayRef<uint8_t> Data,
> + std::function<void(ArrayRef<uint8_t> Arr, uint8_t *Hash)> Hash) {
> + std::vector<ArrayRef<uint8_t>> Chunks = split(Data, 1024 * 1024);
> + std::vector<uint8_t> HashList(Chunks.size() * HashSize);
> +
> + if (Config->Threads)
> + parallel_for_each(Chunks.begin(), Chunks.end(),
> + [&](ArrayRef<uint8_t> &Chunk) {
> + size_t Id = &Chunk - Chunks.data();
> + Hash(Chunk, HashList.data() + Id * HashSize);
> + });
> + else
> + std::for_each(Chunks.begin(), Chunks.end(), [&](ArrayRef<uint8_t> &Chunk) {
> + size_t Id = &Chunk - Chunks.data();
> + Hash(Chunk, HashList.data() + Id * HashSize);
> + });
>
> - // 64-bit xxhash
> - uint64_t Hash = xxHash64(toStringRef(Buf));
> - write64<E>(this->getOutputLoc(Buf.begin()) + 16, Hash);
> + Hash(HashList, this->getOutputLoc((uint8_t *)Data.begin()) + 16);
> +}
> +
> +template <class ELFT>
> +void BuildIdFastHash<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
> + computeHash(Buf, [](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
> + uint64_t Hash = xxHash64(toStringRef(Arr));
> + write64<ELFT::TargetEndianness>(Dest, Hash);
> + });
> }
>
> template <class ELFT>
> void BuildIdMd5<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
> - MD5 Hash;
> - Hash.update(Buf);
> - MD5::MD5Result Res;
> - Hash.final(Res);
> - memcpy(this->getOutputLoc(Buf.begin()) + 16, Res, 16);
> + computeHash(Buf, [&](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
> + MD5 Hash;
> + Hash.update(Arr);
> + MD5::MD5Result Res;
> + Hash.final(Res);
> + memcpy(Dest, Res, HashSize);
> + });
> }
>
> template <class ELFT>
> void BuildIdSha1<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
> - SHA1 Hash;
> - Hash.update(Buf);
> - memcpy(this->getOutputLoc(Buf.begin()) + 16, Hash.final().data(), 20);
> + computeHash(Buf, [&](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
> + SHA1 Hash;
> + Hash.update(Arr);
> + memcpy(Dest, Hash.final().data(), HashSize);
> + });
> }
>
> template <class ELFT>
>
> Modified: lld/trunk/ELF/SyntheticSections.h
> URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/SyntheticSections.h?rev=286061&r1=286060&r2=286061&view=diff
> ==============================================================================
> --- lld/trunk/ELF/SyntheticSections.h (original)
> +++ lld/trunk/ELF/SyntheticSections.h Sun Nov 6 01:42:55 2016
> @@ -38,6 +38,12 @@ public:
> protected:
> BuildIdSection(size_t HashSize);
> std::vector<uint8_t> Buf;
> +
> + void
> + computeHash(llvm::ArrayRef<uint8_t> Buf,
> + std::function<void(ArrayRef<uint8_t> Arr, uint8_t *Hash)> Hash);
> +
> + size_t HashSize;
> };
>
> template <class ELFT>
>
> Modified: lld/trunk/test/ELF/build-id.s
> URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/build-id.s?rev=286061&r1=286060&r2=286061&view=diff
> ==============================================================================
> --- lld/trunk/test/ELF/build-id.s (original)
> +++ lld/trunk/test/ELF/build-id.s Sun Nov 6 01:42:55 2016
> @@ -16,6 +16,14 @@
> # RUN: ld.lld --build-id=md5 --build-id=none %t -o %t2
> # RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=NONE %s
>
> +## Multithreaded cases:
> +# RUN: ld.lld --build-id -threads %t -o %t2
> +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=DEFAULT %s
> +# RUN: ld.lld --build-id=md5 -threads %t -o %t2
> +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=MD5 %s
> +# RUN: ld.lld --build-id=sha1 -threads %t -o %t2
> +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=SHA1 %s
> +
> .globl _start
> _start:
> nop
> @@ -26,12 +34,19 @@ _start:
> # DEFAULT: Contents of section .note.test:
> # DEFAULT: Contents of section .note.gnu.build-id:
> # DEFAULT-NEXT: 04000000 08000000 03000000 474e5500 ............GNU.
> +# DEFAULT-NEXT: ab
>
> # MD5: Contents of section .note.gnu.build-id:
> # MD5-NEXT: 04000000 10000000 03000000 474e5500 ............GNU.
> +# MD5-NEXT: 29
>
> # SHA1: Contents of section .note.gnu.build-id:
> # SHA1-NEXT: 04000000 14000000 03000000 474e5500 ............GNU.
> +# SHA1-NEXT: b1
> +
> +# TREE: Contents of section .note.gnu.build-id:
> +# TREE-NEXT: 04000000 14000000 03000000 474e5500 ............GNU.
> +# TREE-NEXT: 18
>
> # UUID: Contents of section .note.gnu.build-id:
> # UUID-NEXT: 04000000 10000000 03000000 474e5500 ............GNU.
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list