<div dir="ltr"><div class="gmail_extra"><div class="gmail_quote">On Tue, Nov 15, 2016 at 11:32 AM, Rafael Espíndola <span dir="ltr"><<a href="mailto:rafael.espindola@gmail.com" target="_blank">rafael.espindola@gmail.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Have you benchmarked this on windows? I was about to benchmark this on<br>
linux but noticed that parallel_for_each does nothing there since it<br>
uses a thread for each 1024 elements, so in this case it would only<br>
use more than one thread if the file was more than a GiB.<br>
<br>
Any idea why parallel_for_each is implemented that way? It would seem<br>
more natural to create a task for each element and let the caller<br>
create more coarse elements if desired.<br></blockquote><div><br></div><div>I think you can change the code. Our task is fairly coarse, like computing a cryptic hash for a 1 MB blob, or copy & apply relocations for a section, so one vector element can be one task unit.</div><div><br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">
Cheers,<br>
Rafael<br>
<br>
<br>
On 6 November 2016 at 02:42, George Rimar via llvm-commits<br>
<div class="HOEnZb"><div class="h5"><<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a>> wrote:<br>
> Author: grimar<br>
> Date: Sun Nov 6 01:42:55 2016<br>
> New Revision: 286061<br>
><br>
> URL: <a href="http://llvm.org/viewvc/llvm-project?rev=286061&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project?rev=286061&view=rev</a><br>
> Log:<br>
> [ELF] - Implemented threaded --build-id computation<br>
><br>
> Patch switches computing of --build-id hash to tree.<br>
><br>
> This is the way when input data is splitted by chunks,<br>
> hash is computed for each one in threaded/non-threaded way.<br>
> At the end hash is conputed for result tree.<br>
><br>
> With or without -threads the result hash is the same.<br>
><br>
> Differential revision: <a href="https://reviews.llvm.org/D26199" rel="noreferrer" target="_blank">https://reviews.llvm.org/<wbr>D26199</a><br>
><br>
> Modified:<br>
> lld/trunk/ELF/<wbr>SyntheticSections.cpp<br>
> lld/trunk/ELF/<wbr>SyntheticSections.h<br>
> lld/trunk/test/ELF/build-id.s<br>
><br>
> Modified: lld/trunk/ELF/<wbr>SyntheticSections.cpp<br>
> URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/SyntheticSections.cpp?rev=286061&r1=286060&r2=286061&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project/lld/trunk/ELF/<wbr>SyntheticSections.cpp?rev=<wbr>286061&r1=286060&r2=286061&<wbr>view=diff</a><br>
> ==============================<wbr>==============================<wbr>==================<br>
> --- lld/trunk/ELF/<wbr>SyntheticSections.cpp (original)<br>
> +++ lld/trunk/ELF/<wbr>SyntheticSections.cpp Sun Nov 6 01:42:55 2016<br>
> @@ -23,6 +23,7 @@<br>
> #include "Strings.h"<br>
> #include "SymbolTable.h"<br>
><br>
> +#include "lld/Core/Parallel.h"<br>
> #include "llvm/Support/Endian.h"<br>
> #include "llvm/Support/MD5.h"<br>
> #include "llvm/Support/<wbr>RandomNumberGenerator.h"<br>
> @@ -91,7 +92,8 @@ InterpSection<ELFT>::<wbr>InterpSection()<br>
> template <class ELFT><br>
> BuildIdSection<ELFT>::<wbr>BuildIdSection(size_t HashSize)<br>
> : InputSection<ELFT>(SHF_ALLOC, SHT_NOTE, 1, ArrayRef<uint8_t>(),<br>
> - ".note.gnu.build-id") {<br>
> + ".note.gnu.build-id"),<br>
> + HashSize(HashSize) {<br>
> this->Live = true;<br>
><br>
> Buf.resize(16 + HashSize);<br>
> @@ -108,29 +110,66 @@ uint8_t *BuildIdSection<ELFT>::<wbr>getOutput<br>
> return Start + this->OutSec->getFileOffset() + this->OutSecOff;<br>
> }<br>
><br>
> +static std::vector<ArrayRef<uint8_t>> split(ArrayRef<uint8_t> Arr,<br>
> + size_t ChunkSize) {<br>
> + std::vector<ArrayRef<uint8_t>> Ret;<br>
> + while (Arr.size() > ChunkSize) {<br>
> + Ret.push_back(Arr.take_front(<wbr>ChunkSize));<br>
> + Arr = Arr.drop_front(ChunkSize);<br>
> + }<br>
> + if (!Arr.empty())<br>
> + Ret.push_back(Arr);<br>
> + return Ret;<br>
> +}<br>
> +<br>
> template <class ELFT><br>
> -void BuildIdFastHash<ELFT>::<wbr>writeBuildId(MutableArrayRef<<wbr>uint8_t> Buf) {<br>
> - const endianness E = ELFT::TargetEndianness;<br>
> +void BuildIdSection<ELFT>::<wbr>computeHash(<br>
> + llvm::ArrayRef<uint8_t> Data,<br>
> + std::function<void(ArrayRef<<wbr>uint8_t> Arr, uint8_t *Hash)> Hash) {<br>
> + std::vector<ArrayRef<uint8_t>> Chunks = split(Data, 1024 * 1024);<br>
> + std::vector<uint8_t> HashList(Chunks.size() * HashSize);<br>
> +<br>
> + if (Config->Threads)<br>
> + parallel_for_each(Chunks.<wbr>begin(), Chunks.end(),<br>
> + [&](ArrayRef<uint8_t> &Chunk) {<br>
> + size_t Id = &Chunk - Chunks.data();<br>
> + Hash(Chunk, HashList.data() + Id * HashSize);<br>
> + });<br>
> + else<br>
> + std::for_each(Chunks.begin(), Chunks.end(), [&](ArrayRef<uint8_t> &Chunk) {<br>
> + size_t Id = &Chunk - Chunks.data();<br>
> + Hash(Chunk, HashList.data() + Id * HashSize);<br>
> + });<br>
><br>
> - // 64-bit xxhash<br>
> - uint64_t Hash = xxHash64(toStringRef(Buf));<br>
> - write64<E>(this->getOutputLoc(<wbr>Buf.begin()) + 16, Hash);<br>
> + Hash(HashList, this->getOutputLoc((uint8_t *)Data.begin()) + 16);<br>
> +}<br>
> +<br>
> +template <class ELFT><br>
> +void BuildIdFastHash<ELFT>::<wbr>writeBuildId(MutableArrayRef<<wbr>uint8_t> Buf) {<br>
> + computeHash(Buf, [](ArrayRef<uint8_t> Arr, uint8_t *Dest) {<br>
> + uint64_t Hash = xxHash64(toStringRef(Arr));<br>
> + write64<ELFT::<wbr>TargetEndianness>(Dest, Hash);<br>
> + });<br>
> }<br>
><br>
> template <class ELFT><br>
> void BuildIdMd5<ELFT>::<wbr>writeBuildId(MutableArrayRef<<wbr>uint8_t> Buf) {<br>
> - MD5 Hash;<br>
> - Hash.update(Buf);<br>
> - MD5::MD5Result Res;<br>
> - Hash.final(Res);<br>
> - memcpy(this->getOutputLoc(Buf.<wbr>begin()) + 16, Res, 16);<br>
> + computeHash(Buf, [&](ArrayRef<uint8_t> Arr, uint8_t *Dest) {<br>
> + MD5 Hash;<br>
> + Hash.update(Arr);<br>
> + MD5::MD5Result Res;<br>
> + Hash.final(Res);<br>
> + memcpy(Dest, Res, HashSize);<br>
> + });<br>
> }<br>
><br>
> template <class ELFT><br>
> void BuildIdSha1<ELFT>::<wbr>writeBuildId(MutableArrayRef<<wbr>uint8_t> Buf) {<br>
> - SHA1 Hash;<br>
> - Hash.update(Buf);<br>
> - memcpy(this->getOutputLoc(Buf.<wbr>begin()) + 16, Hash.final().data(), 20);<br>
> + computeHash(Buf, [&](ArrayRef<uint8_t> Arr, uint8_t *Dest) {<br>
> + SHA1 Hash;<br>
> + Hash.update(Arr);<br>
> + memcpy(Dest, Hash.final().data(), HashSize);<br>
> + });<br>
> }<br>
><br>
> template <class ELFT><br>
><br>
> Modified: lld/trunk/ELF/<wbr>SyntheticSections.h<br>
> URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/SyntheticSections.h?rev=286061&r1=286060&r2=286061&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project/lld/trunk/ELF/<wbr>SyntheticSections.h?rev=<wbr>286061&r1=286060&r2=286061&<wbr>view=diff</a><br>
> ==============================<wbr>==============================<wbr>==================<br>
> --- lld/trunk/ELF/<wbr>SyntheticSections.h (original)<br>
> +++ lld/trunk/ELF/<wbr>SyntheticSections.h Sun Nov 6 01:42:55 2016<br>
> @@ -38,6 +38,12 @@ public:<br>
> protected:<br>
> BuildIdSection(size_t HashSize);<br>
> std::vector<uint8_t> Buf;<br>
> +<br>
> + void<br>
> + computeHash(llvm::ArrayRef<<wbr>uint8_t> Buf,<br>
> + std::function<void(ArrayRef<<wbr>uint8_t> Arr, uint8_t *Hash)> Hash);<br>
> +<br>
> + size_t HashSize;<br>
> };<br>
><br>
> template <class ELFT><br>
><br>
> Modified: lld/trunk/test/ELF/build-id.s<br>
> URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/build-id.s?rev=286061&r1=286060&r2=286061&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project/lld/trunk/test/ELF/<wbr>build-id.s?rev=286061&r1=<wbr>286060&r2=286061&view=diff</a><br>
> ==============================<wbr>==============================<wbr>==================<br>
> --- lld/trunk/test/ELF/build-id.s (original)<br>
> +++ lld/trunk/test/ELF/build-id.s Sun Nov 6 01:42:55 2016<br>
> @@ -16,6 +16,14 @@<br>
> # RUN: ld.lld --build-id=md5 --build-id=none %t -o %t2<br>
> # RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=NONE %s<br>
><br>
> +## Multithreaded cases:<br>
> +# RUN: ld.lld --build-id -threads %t -o %t2<br>
> +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=DEFAULT %s<br>
> +# RUN: ld.lld --build-id=md5 -threads %t -o %t2<br>
> +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=MD5 %s<br>
> +# RUN: ld.lld --build-id=sha1 -threads %t -o %t2<br>
> +# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=SHA1 %s<br>
> +<br>
> .globl _start<br>
> _start:<br>
> nop<br>
> @@ -26,12 +34,19 @@ _start:<br>
> # DEFAULT: Contents of section .note.test:<br>
> # DEFAULT: Contents of section .note.gnu.build-id:<br>
> # DEFAULT-NEXT: 04000000 08000000 03000000 474e5500 ............GNU.<br>
> +# DEFAULT-NEXT: ab<br>
><br>
> # MD5: Contents of section .note.gnu.build-id:<br>
> # MD5-NEXT: 04000000 10000000 03000000 474e5500 ............GNU.<br>
> +# MD5-NEXT: 29<br>
><br>
> # SHA1: Contents of section .note.gnu.build-id:<br>
> # SHA1-NEXT: 04000000 14000000 03000000 474e5500 ............GNU.<br>
> +# SHA1-NEXT: b1<br>
> +<br>
> +# TREE: Contents of section .note.gnu.build-id:<br>
> +# TREE-NEXT: 04000000 14000000 03000000 474e5500 ............GNU.<br>
> +# TREE-NEXT: 18<br>
><br>
> # UUID: Contents of section .note.gnu.build-id:<br>
> # UUID-NEXT: 04000000 10000000 03000000 474e5500 ............GNU.<br>
><br>
><br>
> ______________________________<wbr>_________________<br>
> llvm-commits mailing list<br>
> <a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a><br>
> <a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/<wbr>mailman/listinfo/llvm-commits</a><br>
</div></div></blockquote></div><br></div></div>