[llvm] 1a00929 - Build reproducible tarballs for releases

[Aaron Puchert via llvm-commits]

Author: Aaron Puchert
Date: 2020-11-22T20:51:58+01:00
New Revision: 1a009296a4e9a50e85908f9141c3c1ea860d73e4

URL: https://github.com/llvm/llvm-project/commit/1a009296a4e9a50e85908f9141c3c1ea860d73e4
DIFF: https://github.com/llvm/llvm-project/commit/1a009296a4e9a50e85908f9141c3c1ea860d73e4.diff

LOG: Build reproducible tarballs for releases

Currently the tarballs contain superfluous metadata, like the user name
of the packager and via Pax headers even the PID of the tar process that
packaged the files. We build the monorepo projects directly from the git
repo using "git archive" and for the test-suite we add some flags as
recommended by https://reproducible-builds.org/docs/archives/. We don't
use numeric owners though to be compatible with "git archive".

The advantage of "git archive" is that the releaser doesn't have to
download the tar ball and extract it, rather the archive is built
directly from the repository. This is probably what GitHub uses
internally to produce the tarballs, so I wouldn't expect a difference.

Reviewed By: tstellar

Differential Revision: https://reviews.llvm.org/D91494

Added: 
    

Modified: 
    llvm/utils/release/export.sh

Removed: 
    


################################################################################
diff  --git a/llvm/utils/release/export.sh b/llvm/utils/release/export.sh
index 3ffd7e78dd63..0c76ed047081 100755
--- a/llvm/utils/release/export.sh
+++ b/llvm/utils/release/export.sh
@@ -13,7 +13,7 @@
 
 set -e
 
-projects="llvm clang test-suite compiler-rt libcxx libcxxabi libclc clang-tools-extra polly lldb lld openmp libunwind flang"
+projects="llvm clang compiler-rt libcxx libcxxabi libclc clang-tools-extra polly lldb lld openmp libunwind flang"
 
 release=""
 rc=""
@@ -37,26 +37,34 @@ export_sources() {
         tag="$tag-$rc"
     fi
 
-    llvm_src_dir=llvm-project-$release$rc
-    mkdir -p $llvm_src_dir
+    llvm_src_dir=$(readlink -f $(dirname "$(readlink -f "$0")")/../../..)
+    [ -d $llvm_src_dir/.git ] || ( echo "No git repository at $llvm_src_dir" ; exit 1 )
 
     echo $tag
-    echo "Fetching LLVM project source ..."
-    curl -L https://github.com/llvm/llvm-project/archive/$tag.tar.gz | \
-        tar -C $llvm_src_dir --strip-components=1 -xzf -
+    target_dir=$(pwd)
 
     echo "Creating tarball for llvm-project ..."
-    tar -cJf llvm-project-$release$rc.tar.xz $llvm_src_dir
+    pushd $llvm_src_dir/
+    git archive --prefix=llvm-project-$release$rc.src/ $tag . | xz >$target_dir/llvm-project-$release$rc.src.tar.xz
+    popd
 
-    echo "Fetching LLVM test-suite source ..."
-    mkdir -p $llvm_src_dir/test-suite
-    curl -L https://github.com/llvm/test-suite/archive/$tag.tar.gz | \
-        tar -C $llvm_src_dir/test-suite --strip-components=1 -xzf -
+    if [ ! -d test-suite-$release$rc.src ]
+    then
+      echo "Fetching LLVM test-suite source ..."
+      mkdir -p test-suite-$release$rc.src
+      curl -L https://github.com/llvm/test-suite/archive/$tag.tar.gz | \
+          tar -C test-suite-$release$rc.src --strip-components=1 -xzf -
+    fi
+    echo "Creating tarball for test-suite ..."
+    tar --sort=name --owner=0 --group=0 \
+        --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \
+        -cJf test-suite-$release$rc.src.tar.xz test-suite-$release$rc.src
 
     for proj in $projects; do
         echo "Creating tarball for $proj ..."
-        mv $llvm_src_dir/$proj $llvm_src_dir/$proj-$release$rc.src
-        tar -C $llvm_src_dir -cJf $proj-$release$rc.src.tar.xz $proj-$release$rc.src
+        pushd $llvm_src_dir/$proj
+        git archive --prefix=$proj-$release$rc.src/ $tag . | xz >$target_dir/$proj-$release$rc.src.tar.xz
+        popd
     done
 }