[lld] 74858f3 - Add Nix recipe for collecting linker reproducers. (#145789)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 9 14:02:02 PDT 2025
Author: Peter Collingbourne
Date: 2025-10-09T14:01:57-07:00
New Revision: 74858f366bad87d24f41819b015bcc1043daead7
URL: https://github.com/llvm/llvm-project/commit/74858f366bad87d24f41819b015bcc1043daead7
DIFF: https://github.com/llvm/llvm-project/commit/74858f366bad87d24f41819b015bcc1043daead7.diff
LOG: Add Nix recipe for collecting linker reproducers. (#145789)
As proposed in:
https://discourse.llvm.org/t/improving-the-reproducibility-of-linker-benchmarking/86057
This is a Nix recipe for collecting reproducers for benchmarking
purposes
in a reproducible way. It works by injecting a linker wrapper that
embeds
a reproducer tarball into a non-allocated section of every linked
object,
which generally causes them to be smuggled out of the build tree in a
section of the final binaries.
It may be used in conjunction with the script lld/utils/run_benchmark.py
to measure the relative performance of linker changes or compare
the performance of different linkers.
Added:
lld/utils/speed-test-reproducers/collect.nix
lld/utils/speed-test-reproducers/ld-wrapper.sh
Modified:
Removed:
################################################################################
diff --git a/lld/utils/speed-test-reproducers/collect.nix b/lld/utils/speed-test-reproducers/collect.nix
new file mode 100644
index 0000000000000..ab5f5eee1aea1
--- /dev/null
+++ b/lld/utils/speed-test-reproducers/collect.nix
@@ -0,0 +1,152 @@
+#===-----------------------------------------------------------------------===//
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===//
+#
+# This is a Nix recipe for collecting reproducers for benchmarking purposes in a
+# reproducible way. It works by injecting a linker wrapper that embeds a
+# reproducer tarball into a non-allocated section of every linked object, which
+# generally causes them to be smuggled out of the build tree in a section of the
+# final binaries. In principle, this technique should let us collect reproducers
+# from any project packaged by Nix without project-specific knowledge, but as
+# you can see below, many interesting ones need a few hacks.
+#
+# If you have Nix installed, you can build the reproducers with the following
+# command:
+#
+# TMPDIR=/var/tmp nix-build -j6 --log-format bar collect.nix
+#
+# This will result in building several large projects including Chromium and
+# Firefox, which will take some time, and it will also build most of the
+# dependencies for non-native targets. Eventually you will get a result
+# directory containing all the reproducers.
+#
+# The following projects have been tested successfully:
+# - chrome (native only, cross builds fail building the qtbase dependency)
+# - firefox (all targets)
+# - linux-kernel (all targets, requires patched nixpkgs)
+# - ladybird (native only, same problem as chromium)
+# - llvm (all targets)
+
+{
+ nixpkgs ? fetchTarball "https://github.com/NixOS/nixpkgs/archive/992f916556fcfaa94451ebc7fc6e396134bbf5b1.tar.gz",
+ system ? builtins.currentSystem,
+}:
+let
+ reproducerPkgs =
+ crossSystem:
+ let
+ pkgsCross = import nixpkgs { inherit crossSystem; };
+ # Wraps the given stdenv and lld package into a variant that collects
+ # the reproducer and builds with debug info.
+ reproducerCollectingStdenv =
+ stdenv: lld:
+ let
+ bintools = stdenv.cc.bintools.override {
+ extraBuildCommands = ''
+ wrap ${stdenv.cc.targetPrefix}nix-wrap-lld ${pkgsCross.path}/pkgs/build-support/bintools-wrapper/ld-wrapper.sh ${lld}/bin/ld.lld
+ export lz4=${pkgsCross.lib.getBin pkgsCross.buildPackages.lz4}/bin/lz4
+ substituteAll ${./ld-wrapper.sh} $out/bin/${stdenv.cc.targetPrefix}ld
+ chmod +x $out/bin/${stdenv.cc.targetPrefix}ld
+ substituteAll ${./ld-wrapper.sh} $out/bin/${stdenv.cc.targetPrefix}ld.lld
+ chmod +x $out/bin/${stdenv.cc.targetPrefix}ld.lld
+ '';
+ };
+ in
+ pkgsCross.withCFlags [ "-g1" ] (
+ stdenv.override (old: {
+ allowedRequisites = null;
+ cc = stdenv.cc.override { inherit bintools; };
+ })
+ );
+ withReproducerCollectingStdenv =
+ pkg:
+ pkg.override {
+ stdenv = reproducerCollectingStdenv pkgsCross.stdenv pkgsCross.buildPackages.lld;
+ };
+ withReproducerCollectingClangStdenv =
+ pkg:
+ pkg.override {
+ clangStdenv = reproducerCollectingStdenv pkgsCross.clangStdenv pkgsCross.buildPackages.lld;
+ };
+ in
+ {
+ # For benchmarking the linker we want to disable LTO as otherwise we would
+ # just be benchmarking the LLVM optimizer. Also, we generally want the
+ # package to use the regular stdenv in order to simplify wrapping it.
+ # Firefox normally uses the rustc stdenv but uses the regular one if
+ # LTO is disabled so we kill two birds with one stone by disabling it.
+ # Chromium uses the rustc stdenv unconditionally so we need the stuff
+ # below to make sure that it finds our wrapped stdenv.
+ chrome =
+ (pkgsCross.chromium.override {
+ newScope =
+ extra:
+ pkgsCross.newScope (
+ extra
+ // {
+ pkgsBuildBuild = {
+ pkg-config = pkgsCross.pkgsBuildBuild.pkg-config;
+ rustc = {
+ llvmPackages = rec {
+ stdenv = reproducerCollectingStdenv pkgsCross.pkgsBuildBuild.rustc.llvmPackages.stdenv pkgsCross.pkgsBuildBuild.rustc.llvmPackages.lld;
+ bintools = stdenv.cc.bintools;
+ };
+ };
+ };
+ }
+ );
+ pkgs = {
+ rustc = {
+ llvmPackages = {
+ stdenv = reproducerCollectingStdenv pkgsCross.rustc.llvmPackages.stdenv pkgsCross.rustc.llvmPackages.lld;
+ };
+ };
+ };
+ }).browser.overrideAttrs
+ (old: {
+ configurePhase = old.configurePhase + ''
+ echo use_thin_lto = false >> out/Release/args.gn
+ echo is_cfi = false >> out/Release/args.gn
+ '';
+ });
+ firefox = (withReproducerCollectingStdenv pkgsCross.firefox-unwrapped).override {
+ ltoSupport = false;
+ pgoSupport = false;
+ };
+ # Doesn't work as-is because the kernel derivation calls the linker
+ # directly instead of the wrapper. See:
+ # https://github.com/NixOS/nixpkgs/blob/d3fdff1631946f3e51318317375d638dae3d6aa2/pkgs/os-specific/linux/kernel/common-flags.nix#L12
+ linux-kernel = (withReproducerCollectingStdenv pkgsCross.linux_latest).dev;
+ ladybird = withReproducerCollectingStdenv pkgsCross.ladybird;
+ llvm = withReproducerCollectingStdenv pkgsCross.llvm;
+ webkitgtk = withReproducerCollectingClangStdenv pkgsCross.webkitgtk;
+ hello = withReproducerCollectingStdenv pkgsCross.hello;
+ };
+ targets = {
+ x86_64 = reproducerPkgs { config = "x86_64-unknown-linux-gnu"; };
+ aarch64 = reproducerPkgs { config = "aarch64-unknown-linux-gnu"; };
+ riscv64 = reproducerPkgs { config = "riscv64-unknown-linux-gnu"; };
+ };
+ pkgs = import nixpkgs { };
+in
+pkgs.runCommand "lld-speed-test" { } ''
+ extract_reproducer() {
+ ${pkgs.coreutils}/bin/mkdir -p $out/$2
+ ${pkgs.llvm}/bin/llvm-objcopy -O binary --only-section=.lld_repro --set-section-flags .lld_repro=alloc $1 - | ${pkgs.gnutar}/bin/tar x -I ${pkgs.lib.getBin pkgs.buildPackages.lz4}/bin/lz4 --strip-components=1 -C $out/$2
+ }
+
+ extract_reproducer ${targets.aarch64.hello}/bin/hello hello-arm64
+ extract_reproducer ${targets.x86_64.hello}/bin/hello hello-x64
+ extract_reproducer ${targets.aarch64.chrome}/libexec/chromium/chromium chrome
+ extract_reproducer ${targets.aarch64.ladybird}/lib/liblagom-web.so ladybird
+ extract_reproducer ${targets.aarch64.firefox}/lib/firefox/libxul.so firefox-arm64
+ extract_reproducer ${targets.x86_64.firefox}/lib/firefox/libxul.so firefox-x64
+ extract_reproducer ${targets.riscv64.firefox}/lib/firefox/libxul.so firefox-riscv64
+ extract_reproducer ${pkgs.lib.getLib targets.aarch64.llvm}/lib/libLLVM.so llvm-arm64
+ extract_reproducer ${pkgs.lib.getLib targets.x86_64.llvm}/lib/libLLVM.so llvm-x64
+ extract_reproducer ${pkgs.lib.getLib targets.riscv64.llvm}/lib/libLLVM.so llvm-riscv6
+''
diff --git a/lld/utils/speed-test-reproducers/ld-wrapper.sh b/lld/utils/speed-test-reproducers/ld-wrapper.sh
new file mode 100755
index 0000000000000..8a19d7e2d87eb
--- /dev/null
+++ b/lld/utils/speed-test-reproducers/ld-wrapper.sh
@@ -0,0 +1,50 @@
+#! @shell@
+
+source @out@/nix-support/utils.bash
+
+expandResponseParams "$@"
+
+output="a.out"
+should_add_repro=true
+newparams=()
+for arg in "${params[@]}"; do
+ case "$arg" in
+ -r|--version)
+ should_add_repro=false
+ ;;
+ *)
+ ;;
+ esac
+ case "$prev" in
+ -o)
+ output="$arg"
+ newparams+=("$arg")
+ ;;
+ *)
+ if [ -e "$arg.nolldrepro" ]; then
+ newparams+=("$arg.nolldrepro")
+ else
+ newparams+=("$arg")
+ fi
+ ;;
+ esac
+ prev="$arg"
+done
+
+export LLD_REPRODUCE="$output.repro.tar"
+if @targetPrefix at nix-wrap-lld "${newparams[@]}"; then
+ if $should_add_repro; then
+ @lz4@ -c "$LLD_REPRODUCE" > "$LLD_REPRODUCE.lz4"
+ mv "$output" "$output.nolldrepro"
+ @targetPrefix at objcopy --add-section ".lld_repro=$LLD_REPRODUCE.lz4" "$output.nolldrepro" "$output"
+ rm -f "$LLD_REPRODUCE.lz4"
+ fi
+ exitcode=0
+else
+ # Some Nix packages don't link with lld so just use bfd instead.
+ @targetPrefix at ld.bfd "${newparams[@]}"
+ exitcode=$?
+fi
+
+rm -f "$LLD_REPRODUCE"
+exit $exitcode
More information about the llvm-commits
mailing list