[llvm] release-binaries: Parallelize steps to reduce total run time (PR #218201)
Tom Stellard via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 03:43:10 PDT 2026
https://github.com/tstellar updated https://github.com/llvm/llvm-project/pull/218201
>From f935c5bbd6b58f3e81d2992e04000a1e2c598239 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 22 Aug 2026 22:40:59 -0700
Subject: [PATCH 1/6] release-binaries: Parallelize steps to reduce total run
time
We can run several of the initial setup steps in parallel and after the
build, we can also run the compression/upload steps in parallel with the
test step.
---
.github/workflows/release-binaries.yml | 109 +++++++++++++++----------
1 file changed, 68 insertions(+), 41 deletions(-)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index dce295770b84c..ff8b37757d87a 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -230,33 +230,17 @@ jobs:
digest: ${{ steps.digest.outputs.digest }}
artifact-id: ${{ steps.artifact-upload.outputs.artifact-id }}
steps:
- # We need to hard code the python library path for Windows, so in order
- # to do that we need to specify a specific python version. It's also
- # good practice to do this on other OSes so the version of python doesn't
- # get changed unexpectedly.
- - uses: actions/setup-python at ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
- with:
- python-version: '3.14.6'
-
- # For some reason this is needed on Windows or else the build system can't find python3.lib.
- - name: Setup Python library path
- if: runner.os == 'Windows'
- run: |
- echo "LIB=$env:LIB;C:\hostedtoolcache\windows\Python\3.14.6\$($env:RUNNER_ARCH.ToLower())\libs" >> $env:GITHUB_ENV
-
+ # Start of parallel block
+ # Wix is not installed by default on Windows ARM64
- name: Setup crlf
+ id: setup-crlf
+ background: true
if: runner.os == 'Windows'
run: |
git config --global core.autocrlf false
- - name: Checkout LLVM
- uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- persist-credentials: false
- ref: ${{ needs.prepare.outputs.ref }}
-
- name: Set Build Prefix
- id: setup-stage
+ background: true
shell: bash
run: |
build_prefix=`pwd`
@@ -266,6 +250,45 @@ jobs:
fi
echo "build-prefix=$build_prefix" >> $GITHUB_OUTPUT
+ - name: Install Wix (Windows ARM64)
+ background: true
+ if: runner.os == 'Windows' && runner.arch == 'ARM64'
+ run: |
+ choco install wixtoolset --version 3.14.1.20250415
+ # Add wix install directory to PATH.
+ Split-Path (Get-ChildItem -Path "C:\" -Filter candle.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1).FullName
+
+ # We need to hard code the python library path for Windows, so in order
+ # to do that we need to specify a specific python version. It's also
+ # good practice to do this on other OSes so the version of python doesn't
+ # get changed unexpectedly.
+ - uses: actions/setup-python at ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+ id: setup-python
+ background: true
+ with:
+ python-version: '3.14.6'
+
+ - wait: setup-crlf
+
+ - name: Checkout LLVM
+ background: true
+ uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ persist-credentials: false
+ ref: ${{ needs.prepare.outputs.ref }}
+
+ - wait: setup-python
+
+ # For some reason this is needed on Windows or else the build system can't find python3.lib.
+ - name: Setup Python library path
+ background: true
+ if: runner.os == 'Windows'
+ run: |
+ echo "LIB=$env:LIB;C:\hostedtoolcache\windows\Python\3.14.6\$($env:RUNNER_ARCH.ToLower())\libs" >> $env:GITHUB_ENV
+
+ - wait-all:
+ # End of Parallel block.
+
- name: Configure Linux/MacOS
id: build
if: runner.os != 'Windows'
@@ -291,14 +314,6 @@ jobs:
release_dir=`find "$BUILD_PREFIX/build" -iname 'stage2-bins'`
mv "$release_dir/$RELEASE_BINARY_FILENAME" .
- # Wix is not installed by default on Windows ARM64
- - name: Install Wix (Windows ARM64)
- if: runner.os == 'Windows' && runner.arch == 'ARM64'
- run: |
- choco install wixtoolset --version 3.14.1.20250415
- # Add wix install directory to PATH.
- Split-Path (Get-ChildItem -Path "C:\" -Filter candle.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1).FullName
-
- name: Build Windows
id: build-windows
if: runner.os == 'Windows'
@@ -329,6 +344,8 @@ jobs:
cat S:/llvm/utils/release/llvm_package_$($env:LLVM_VERSION)/build_$($env:BUILD_DIR_SUFFIX)/_CPack_Packages/win64/WIX/wix.log
- name: Create zstd compressed tarball
+ id: zstd
+ background: true
shell: bash
env:
RELEASE_BINARY_FILENAME: ${{ needs.prepare.outputs.release-binary-filename }}
@@ -337,8 +354,27 @@ jobs:
run: |
xz -dc "$RELEASE_BINARY_FILENAME" | "$ZSTD" --ultra -22 -T0 -o "$RELEASE_BINARY_FILENAME_ZSTD"
+ - name: Install LLDB test Python dependencies
+ id: lldb-deps
+ if: runner.os == 'Linux'
+ run: |
+ python -m pip install -r lldb/test/requirements.txt
+
+ - name: Run Tests
+ background: true
+ # These almost always fail so don't let them fail the build and prevent the uploads.
+ if : runner.os != 'Windows'
+ continue-on-error: true
+ env:
+ BUILD_PREFIX: ${{ steps.setup-stage.outputs.build-prefix }}
+ run: |
+ ninja -C "$BUILD_PREFIX/build" stage2-check-all
+
+ - wait: zstd
+
- name: Generate sha256 digest for binaries
id: digest
+ background: true
shell: bash
env:
RELEASE_BINARY_FILENAME: ${{ needs.prepare.outputs.release-binary-filename }}
@@ -354,13 +390,11 @@ jobs:
fi
echo "digest=$(cat $WINDOWS_INSTALLER_FILENAME $RELEASE_BINARY_FILENAME $RELEASE_BINARY_FILENAME_ZSTD | $sha256sum | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
- - name: Install LLDB test Python dependencies
- if: runner.os == 'Linux'
- run: |
- python -m pip install -r lldb/test/requirements.txt
+ - wait: [zstd, digest]
- uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
id: artifact-upload
+ background: true
with:
name: ${{ runner.os }}-${{ runner.arch }}-release-binary
# Due to path differences on Windows when running in bash vs running on node,
@@ -371,14 +405,7 @@ jobs:
${{ needs.prepare.outputs.release-binary-filename-zstd }}
${{ steps.build-windows.outputs.windows-installer-filename }}
- - name: Run Tests
- # These almost always fail so don't let them fail the build and prevent the uploads.
- if : runner.os != 'Windows'
- continue-on-error: true
- env:
- BUILD_PREFIX: ${{ steps.setup-stage.outputs.build-prefix }}
- run: |
- ninja -C "$BUILD_PREFIX/build" stage2-check-all
+ - wait-all:
upload-release-binaries:
name: "Upload Release Binaries"
>From 6fe37a094003f09bf6311a8a587a0854737ab2c0 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 22 Aug 2026 22:55:39 -0700
Subject: [PATCH 2/6] Fix
---
.github/workflows/release-binaries.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index ff8b37757d87a..87534c8f3422d 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -240,6 +240,7 @@ jobs:
git config --global core.autocrlf false
- name: Set Build Prefix
+ id: setup-stage
background: true
shell: bash
run: |
>From 213af82ebcb789e4ab461943301f69e63a67a0f6 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sun, 23 Aug 2026 02:42:32 -0700
Subject: [PATCH 3/6] Re-order wix install
---
.github/workflows/release-binaries.yml | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 87534c8f3422d..e39836312b8d8 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -251,14 +251,6 @@ jobs:
fi
echo "build-prefix=$build_prefix" >> $GITHUB_OUTPUT
- - name: Install Wix (Windows ARM64)
- background: true
- if: runner.os == 'Windows' && runner.arch == 'ARM64'
- run: |
- choco install wixtoolset --version 3.14.1.20250415
- # Add wix install directory to PATH.
- Split-Path (Get-ChildItem -Path "C:\" -Filter candle.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1).FullName
-
# We need to hard code the python library path for Windows, so in order
# to do that we need to specify a specific python version. It's also
# good practice to do this on other OSes so the version of python doesn't
@@ -280,6 +272,14 @@ jobs:
- wait: setup-python
+ - name: Install Wix (Windows ARM64)
+ background: true
+ if: runner.os == 'Windows' && runner.arch == 'ARM64'
+ run: |
+ choco install wixtoolset --version 3.14.1.20250415
+ # Add wix install directory to PATH.
+ Split-Path (Get-ChildItem -Path "C:\" -Filter candle.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1).FullName
+
# For some reason this is needed on Windows or else the build system can't find python3.lib.
- name: Setup Python library path
background: true
>From 9f2e5506fa7380f155844dd68ddae5833e5e6dfc Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sun, 23 Aug 2026 03:04:37 -0700
Subject: [PATCH 4/6] Debug
---
.github/workflows/release-binaries.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index e39836312b8d8..573cb684a4bfa 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -256,6 +256,7 @@ jobs:
# good practice to do this on other OSes so the version of python doesn't
# get changed unexpectedly.
- uses: actions/setup-python at ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+ if: false
id: setup-python
background: true
with:
@@ -272,6 +273,7 @@ jobs:
- wait: setup-python
+ # For some reason this Install Wix step fails if it is run before setup-python.
- name: Install Wix (Windows ARM64)
background: true
if: runner.os == 'Windows' && runner.arch == 'ARM64'
>From 8a6af663e0c2d78e4b3a50fb79be914dec9b9a0f Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sun, 23 Aug 2026 03:24:59 -0700
Subject: [PATCH 5/6] Fix
---
.github/workflows/release-binaries.yml | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 573cb684a4bfa..87534c8f3422d 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -251,12 +251,19 @@ jobs:
fi
echo "build-prefix=$build_prefix" >> $GITHUB_OUTPUT
+ - name: Install Wix (Windows ARM64)
+ background: true
+ if: runner.os == 'Windows' && runner.arch == 'ARM64'
+ run: |
+ choco install wixtoolset --version 3.14.1.20250415
+ # Add wix install directory to PATH.
+ Split-Path (Get-ChildItem -Path "C:\" -Filter candle.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1).FullName
+
# We need to hard code the python library path for Windows, so in order
# to do that we need to specify a specific python version. It's also
# good practice to do this on other OSes so the version of python doesn't
# get changed unexpectedly.
- uses: actions/setup-python at ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
- if: false
id: setup-python
background: true
with:
@@ -273,15 +280,6 @@ jobs:
- wait: setup-python
- # For some reason this Install Wix step fails if it is run before setup-python.
- - name: Install Wix (Windows ARM64)
- background: true
- if: runner.os == 'Windows' && runner.arch == 'ARM64'
- run: |
- choco install wixtoolset --version 3.14.1.20250415
- # Add wix install directory to PATH.
- Split-Path (Get-ChildItem -Path "C:\" -Filter candle.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1).FullName
-
# For some reason this is needed on Windows or else the build system can't find python3.lib.
- name: Setup Python library path
background: true
>From 472415673176021b88d670de66132ac8d6e98699 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sun, 23 Aug 2026 03:42:49 -0700
Subject: [PATCH 6/6] Fix wix
---
.github/workflows/release-binaries.yml | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 87534c8f3422d..cad4a49ecc19a 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -251,14 +251,6 @@ jobs:
fi
echo "build-prefix=$build_prefix" >> $GITHUB_OUTPUT
- - name: Install Wix (Windows ARM64)
- background: true
- if: runner.os == 'Windows' && runner.arch == 'ARM64'
- run: |
- choco install wixtoolset --version 3.14.1.20250415
- # Add wix install directory to PATH.
- Split-Path (Get-ChildItem -Path "C:\" -Filter candle.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1).FullName
-
# We need to hard code the python library path for Windows, so in order
# to do that we need to specify a specific python version. It's also
# good practice to do this on other OSes so the version of python doesn't
@@ -280,6 +272,17 @@ jobs:
- wait: setup-python
+ # For some reason this Install Wix step fails if it is running at the same time as setup-python.
+ # I've tested running it with the setup-python step disabled and it installs fine, but running
+ # it with setup-python causes some kind of issue. That is why we run it after setup-python.
+ - name: Install Wix (Windows ARM64)
+ background: true
+ if: runner.os == 'Windows' && runner.arch == 'ARM64'
+ run: |
+ choco install wixtoolset --version 3.14.1.20250415
+ # Add wix install directory to PATH.
+ Split-Path (Get-ChildItem -Path "C:\" -Filter candle.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1).FullName
+
# For some reason this is needed on Windows or else the build system can't find python3.lib.
- name: Setup Python library path
background: true
More information about the llvm-commits
mailing list