[llvm-branch-commits] [clang] [llvm] release/19.x: Workflow fixes for building release binaries (PR #101791)
Tom Stellard via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 2 21:43:12 PDT 2024
https://github.com/tstellar created https://github.com/llvm/llvm-project/pull/101791
This is a collection of fixes needed for building the release binaries.
>From 3ec814d1bcb7064b907526937ae83f23657394af Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Thu, 1 Aug 2024 07:58:35 -0700
Subject: [PATCH 1/6] workflows: Fix libclc-tests
The old out-of-tree build configuration stopped working and
in tree builds are supported now, so we should use the in
tree configuration. The only downside is we can't run the tests any
more, but at least we will be able to test the build again.
---
.github/workflows/llvm-project-tests.yml | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml
index 0a228c41f354e..17a54be16badc 100644
--- a/.github/workflows/llvm-project-tests.yml
+++ b/.github/workflows/llvm-project-tests.yml
@@ -131,6 +131,7 @@ jobs:
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLDB_INCLUDE_TESTS=OFF \
+ -DLIBCLC_TARGETS_TO_BUILD="amdgcn--;amdgcn--amdhsa;r600--;nvptx--;nvptx64--;nvptx--nvidiacl;nvptx64--nvidiacl" \
-DCMAKE_C_COMPILER_LAUNCHER=sccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
$extra_cmake_args \
@@ -142,8 +143,6 @@ jobs:
env:
LLVM_BUILDDIR: ${{ steps.build-llvm.outputs.llvm-builddir }}
run: |
- # Make sure all of LLVM libraries that llvm-config needs are built.
+ # The libclc tests don't have a generated check target so all we can
+ # do is build it.
ninja -C "$LLVM_BUILDDIR"
- cmake -G Ninja -S libclc -B libclc-build -DLLVM_DIR="$LLVM_BUILDDIR"/lib/cmake/llvm -DLIBCLC_TARGETS_TO_BUILD="amdgcn--;amdgcn--amdhsa;r600--;nvptx--;nvptx64--;nvptx--nvidiacl;nvptx64--nvidiacl"
- ninja -C libclc-build
- ninja -C libclc-build test
>From fdd961d5dbd68ca05e64b726e73a149ae67e680d Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 26 Jul 2024 11:26:34 -0700
Subject: [PATCH 2/6] Build release binaries for multiple targets (#98431)
This adds release binary builds for the 4 platforms currently supported
by the free GitHub Action runners:
* Linux x86_64
* Windows x86_64
* Mac x86_64
* Mac AArch64
The test stages for these are known to fail, but the creating and
upoading of the release binaries should pass.
(cherry picked from commit 247251aee0d4314385a3fea86e31484d3d792ffb)
---
.github/workflows/release-binaries-all.yml | 94 ++++
.../release-binaries-save-stage/action.yml | 38 ++
.../release-binaries-setup-stage/action.yml | 59 +++
.github/workflows/release-binaries.yml | 474 ++++++++++++------
.github/workflows/release-tasks.yml | 10 +
clang/cmake/caches/Release.cmake | 6 +-
6 files changed, 520 insertions(+), 161 deletions(-)
create mode 100644 .github/workflows/release-binaries-all.yml
create mode 100644 .github/workflows/release-binaries-save-stage/action.yml
create mode 100644 .github/workflows/release-binaries-setup-stage/action.yml
diff --git a/.github/workflows/release-binaries-all.yml b/.github/workflows/release-binaries-all.yml
new file mode 100644
index 0000000000000..73c9d96946e33
--- /dev/null
+++ b/.github/workflows/release-binaries-all.yml
@@ -0,0 +1,94 @@
+name: Release Binaries All
+
+permissions:
+ contents: read # Default everything to read-only
+
+on:
+ workflow_dispatch:
+ inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
+ upload:
+ description: 'Upload binaries to the release page'
+ required: true
+ default: false
+ type: boolean
+
+ workflow_call:
+ inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
+ upload:
+ description: 'Upload binaries to the release page'
+ required: true
+ default: false
+ type: boolean
+
+ pull_request:
+ types:
+ - opened
+ - synchronize
+ - reopened
+ # When a PR is closed, we still start this workflow, but then skip
+ # all the jobs, which makes it effectively a no-op. The reason to
+ # do this is that it allows us to take advantage of concurrency groups
+ # to cancel in progress CI jobs whenever the PR is closed.
+ - closed
+ paths:
+ - '.github/workflows/release-binaries-all.yml'
+ - '.github/workflows/release-binaries.yml'
+ - '.github/workflows/release-binaries-setup-stage/*'
+ - '.github/workflows/release-binaries-save-stage/*'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || 'dispatch' }}
+ cancel-in-progress: True
+
+jobs:
+ setup-variables:
+ if: >-
+ (github.event_name != 'pull_request' || github.event.action != 'closed')
+ runs-on: ubuntu-22.04
+ outputs:
+ release-version: ${{ steps.vars.outputs.release-version }}
+ upload: ${{ steps.vars.outputs.upload }}
+ steps:
+ - shell: bash
+ id: vars
+ run: |
+ upload="${{ inputs.upload }}"
+ release_version="${{ inputs.release-version }}"
+ if [ "${{ github.event_name }}" = "pull_request" ]; then
+ upload="false"
+ release_version=""
+ fi
+ echo "release-version=$release_version" >> "$GITHUB_OUTPUT"
+ echo "upload=$upload" >> "$GITHUB_OUTPUT"
+
+ release-binaries-all:
+ name: Build Release Binaries
+ needs:
+ - setup-variables
+ permissions:
+ contents: write # For release uploads
+ id-token: write # For artifact attestations
+ attestations: write # For artifact attestations
+ strategy:
+ fail-fast: false
+ matrix:
+ runs-on:
+ - ubuntu-22.04
+ - windows-2022
+ - macos-13
+ - macos-14
+
+ uses: ./.github/workflows/release-binaries.yml
+ with:
+ release-version: "${{ needs.setup-variables.outputs.release-version }}"
+ upload: ${{ needs.setup-variables.outputs.upload == 'true'}}
+ runs-on: "${{ matrix.runs-on }}"
+
diff --git a/.github/workflows/release-binaries-save-stage/action.yml b/.github/workflows/release-binaries-save-stage/action.yml
new file mode 100644
index 0000000000000..e2f3eeadd15be
--- /dev/null
+++ b/.github/workflows/release-binaries-save-stage/action.yml
@@ -0,0 +1,38 @@
+name: Save Stage
+description: >-
+ Upload the source and binary directories from a build stage so that they
+ can be re-used in the next stage. This action is used to the release
+ binaries workflow into multiple stages to avoid the 6 hour timeout on
+ the GitHub hosted runners.
+inputs:
+ build-prefix:
+ description: "Directory containing the build directory."
+ required: true
+ type: 'string'
+
+runs:
+ using: "composite"
+ steps:
+ # We need to create an archive of the build directory, because it has too
+ # many files to upload.
+ - name: Package Build and Source Directories
+ shell: bash
+ run: |
+ # Windows does not support symlinks, so we need to dereference them.
+ tar --exclude build/ ${{ (runner.os == 'Windows' && '-h') || '' }} -c . | zstd -T0 -c > ../llvm-project.tar.zst
+ mv ../llvm-project.tar.zst .
+ tar -C ${{ inputs.build-prefix }} -c build/ | zstd -T0 -c > build.tar.zst
+
+ - name: Upload Stage 1 Source
+ uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ with:
+ name: ${{ runner.os }}-${{ runner.arch }}-${{ github.job }}-source
+ path: llvm-project.tar.zst
+ retention-days: 2
+
+ - name: Upload Stage 1 Build Dir
+ uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ with:
+ name: ${{ runner.os}}-${{ runner.arch }}-${{ github.job }}-build
+ path: build.tar.zst
+ retention-days: 2
diff --git a/.github/workflows/release-binaries-setup-stage/action.yml b/.github/workflows/release-binaries-setup-stage/action.yml
new file mode 100644
index 0000000000000..f5e5db27e6595
--- /dev/null
+++ b/.github/workflows/release-binaries-setup-stage/action.yml
@@ -0,0 +1,59 @@
+name: Setup Stage
+description: >-
+ Setup the next stage of the release binaries workflow. This sets up the
+ environment correctly for a new stage of the release binaries workflow
+ and also restores the source and build directory from the previous stage.
+
+inputs:
+ previous-artifact:
+ description: >-
+ A unique descriptor for the artifact from the previous stage. This will
+ be used to construct the final artifact pattern, which is:
+ $RUNNER_OS-$RUNNER_ARCH-$PREVIOUS_ARTIFACT-*
+ required: false
+ type: 'string'
+
+outputs:
+ build-prefix:
+ description: "Directory containing the build directory."
+ value: ${{ steps.build-prefix.outputs.build-prefix }}
+
+runs:
+ using: "composite"
+ steps:
+ - name: Install Ninja
+ uses: llvm/actions/install-ninja at 22e9f909d35b50bd1181709564bfe816eaeaae81 # main
+
+ - name: Setup Windows
+ if: startsWith(runner.os, 'Windows')
+ uses: llvm/actions/setup-windows at main
+ with:
+ arch: amd64
+
+ - name: Set Build Prefix
+ id: build-prefix
+ shell: bash
+ run: |
+ build_prefix=`pwd`
+ if [ "${{ runner.os }}" = "Linux" ]; then
+ sudo chown $USER:$USER /mnt/
+ build_prefix=/mnt/
+ fi
+ echo "build-prefix=$build_prefix" >> $GITHUB_OUTPUT
+
+ - name: Download Previous Stage Artifact
+ if: ${{ inputs.previous-artifact }}
+ id: download
+ uses: actions/download-artifact at 6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+ with:
+ pattern: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.previous-artifact }}-*
+ merge-multiple: true
+
+ - name: Unpack Artifact
+ if: ${{ steps.download.outputs.download-path }}
+ shell: bash
+ run: |
+ tar --zstd -xf llvm-project.tar.zst
+ rm llvm-project.tar.zst
+ tar --zstd -C ${{ steps.build-prefix.outputs.build-prefix}} -xf build.tar.zst
+ rm build.tar.zst
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 7de4d00334d14..5f939ba5bbe2f 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -5,28 +5,38 @@ on:
inputs:
release-version:
description: 'Release Version'
- required: true
+ required: false
type: string
upload:
description: 'Upload binaries to the release page'
required: true
default: false
type: boolean
+ runs-on:
+ description: "Runner to use for the build"
+ required: true
+ type: choice
+ options:
+ - ubuntu-22.04
+ - windows-2022
+ - macos-13
+ - macos-14
workflow_call:
inputs:
release-version:
description: 'Release Version'
- required: true
+ required: false
type: string
upload:
description: 'Upload binaries to the release page'
required: true
default: false
type: boolean
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 8 1 * *'
+ runs-on:
+ description: "Runner to use for the build"
+ required: true
+ type: string
permissions:
contents: read # Default everything to read-only
@@ -34,30 +44,39 @@ permissions:
jobs:
prepare:
name: Prepare to build binaries
- runs-on: ubuntu-22.04
+ runs-on: ${{ inputs.runs-on }}
if: github.repository == 'llvm/llvm-project'
outputs:
release-version: ${{ steps.vars.outputs.release-version }}
ref: ${{ steps.vars.outputs.ref }}
upload: ${{ steps.vars.outputs.upload }}
+ target-cmake-flags: ${{ steps.vars.outputs.target-cmake-flags }}
+ build-flang: ${{ steps.vars.outputs.build-flang }}
+ enable-pgo: ${{ steps.vars.outputs.enable-pgo }}
+ release-binary-basename: ${{ steps.vars.outputs.release-binary-basename }}
+ release-binary-filename: ${{ steps.vars.outputs.release-binary-filename }}
steps:
- name: Checkout LLVM
uses: actions/checkout at b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Install Dependencies
+ shell: bash
run: |
pip install --require-hashes -r ./llvm/utils/git/requirements.txt
- name: Check Permissions
+ if: github.event_name != 'pull_request'
env:
GITHUB_TOKEN: ${{ github.token }}
USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
+ shell: bash
run: |
./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} --user-token "$USER_TOKEN" check-permissions
- name: Collect Variables
id: vars
+ shell: bash
# In order for the test-release.sh script to run correctly, the LLVM
# source needs to be at the following location relative to the build dir:
# | X.Y.Z-rcN | ./rcN/llvm-project
@@ -67,242 +86,377 @@ jobs:
# | X.Y.Z-rcN | -rc N -test-asserts
# | X.Y.Z | -final
run: |
- tag="${{ github.ref_name }}"
trimmed=$(echo ${{ inputs.release-version }} | xargs)
- [[ "$trimmed" != "" ]] && tag="llvmorg-$trimmed"
- if [ "$tag" = "main" ]; then
- # If tag is main, then we've been triggered by a scheduled so pass so
- # use the head commit as the tag.
- tag=`git rev-parse HEAD`
+ if [ -n "$trimmed" ]; then
+ release_version="$trimmed"
+ ref="llvmorg-$release_version"
+ else
+ release_version="${{ (github.event_name == 'pull_request' && format('PR{0}', github.event.pull_request.number)) || 'CI'}}-${{ github.sha }}"
+ ref=${{ github.sha }}
fi
if [ -n "${{ inputs.upload }}" ]; then
upload="${{ inputs.upload }}"
else
upload="false"
fi
- bash .github/workflows/set-release-binary-outputs.sh "$tag" "$upload"
+ echo "release-version=$release_version">> $GITHUB_OUTPUT
+ echo "ref=$ref" >> $GITHUB_OUTPUT
+ echo "upload=$upload" >> $GITHUB_OUTPUT
+
+ release_binary_basename="LLVM-$release_version-${{ runner.os }}-${{ runner.arch }}"
+ echo "release-binary-basename=$release_binary_basename" >> $GITHUB_OUTPUT
+ echo "release-binary-filename=$release_binary_basename.tar.xz" >> $GITHUB_OUTPUT
+
+ # Detect necessary CMake flags
+ target="${{ runner.os }}-${{ runner.arch }}"
+ echo "enable-pgo=false" >> $GITHUB_OUTPUT
+ target_cmake_flags="-DLLVM_RELEASE_ENABLE_PGO=OFF"
+ # The macOS builds try to cross compile some libraries so we need to
+ # add extra CMake args to disable them.
+ # See https://github.com/llvm/llvm-project/issues/99767
+ if [ "${{ runner.os }}" = "macOS" ]; then
+ target_cmake_flags="$target_cmake_flags -DBOOTSTRAP_COMPILER_RT_ENABLE_IOS=OFF"
+ if [ "${{ runner.arch }}" = "ARM64" ]; then
+ arches=arm64
+ else
+ arches=x86_64
+ fi
+ target_cmake_flags="$target_cmake_flags -DBOOTSTRAP_DARWIN_osx_ARCHS=$arches -DBOOTSTRAP_DARWIN_osx_BUILTIN_ARCHS=$arches"
+ fi
- build-stage1-linux:
- name: "Build Stage 1 Linux"
+ # x86 macOS and x86 Windows have trouble building flang, so disable it.
+ # Windows: https://github.com/llvm/llvm-project/issues/100202
+ # macOS: 'rebase opcodes terminated early at offset 1 of 80016' when building __fortran_builtins.mod
+ build_flang="true"
+
+ if [ "$target" = "Windows-X64" ]; then
+ target_cmake_flags="$target_cmake_flags -DLLVM_RELEASE_ENABLE_PROJECTS=\"clang;lld;lldb;clang-tools-extra;bolt;polly;mlir\""
+ build_flang="false"
+ fi
+
+ if [ "${{ runner.os }}" = "Windows" ]; then
+ # The build times out on Windows, so we need to disable LTO.
+ target_cmake_flags="$target_cmake_flags -DLLVM_RELEASE_ENABLE_LTO=OFF"
+ fi
+
+ echo "target-cmake-flags=$target_cmake_flags" >> $GITHUB_OUTPUT
+ echo "build-flang=$build_flang" >> $GITHUB_OUTPUT
+
+ build-stage1:
+ name: "Build Stage 1"
needs: prepare
- runs-on: ubuntu-22.04
if: github.repository == 'llvm/llvm-project'
+ runs-on: ${{ inputs.runs-on }}
steps:
- name: Checkout LLVM
uses: actions/checkout at b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ needs.prepare.outputs.ref }}
-
- - name: Install Ninja
- uses: llvm/actions/install-ninja at 22e9f909d35b50bd1181709564bfe816eaeaae81 # main
+ - name: Debug - Move actions
+ if: github.event_name != 'pull_request'
+ shell: bash
+ run: |
+ cd .github/workflows
+ for d in release-binaries-setup-stage release-binaries-save-stage; do
+ mkdir $d
+ pushd $d
+ curl -O -L https://raw.githubusercontent.com/tstellar/llvm-project/main/.github/workflows/$d/action.yml
+ popd
+ done
- name: Setup sccache
uses: hendrikmuhs/ccache-action at ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9
with:
- max-size: 250M
- key: sccache-${{ runner.os }}-release
+ # Default to 2G to workaround: https://github.com/hendrikmuhs/ccache-action/issues/174
+ max-size: 2G
+ key: sccache-${{ runner.os }}-${{ runner.arch }}-release
variant: sccache
- - name: Build Stage 1 Clang
- run: |
- sudo chown $USER:$USER /mnt/
- cmake -G Ninja -C clang/cmake/caches/Release.cmake -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -S llvm -B /mnt/build
- ninja -v -C /mnt/build
+ - name: Setup Stage
+ id: setup-stage
+ uses: ./.github/workflows/release-binaries-setup-stage
- # We need to create an archive of the build directory, because it has too
- # many files to upload.
- - name: Package Build and Source Directories
+ - name: Build Stage 1 Clang
+ id: build
+ shell: bash
run: |
- tar -c . | zstd -T0 -c > llvm-project.tar.zst
- tar -C /mnt/ -c build/ | zstd -T0 -c > build.tar.zst
-
- - name: Upload Stage 1 Source
- uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ # There were some issues on the ARM64 MacOS runners with trying to build x86 object,
+ # so we need to set some extra cmake flags to disable this.
+ cmake -G Ninja -S llvm -B ${{ steps.setup-stage.outputs.build-prefix }}/build \
+ ${{ needs.prepare.outputs.target-cmake-flags }} \
+ -C clang/cmake/caches/Release.cmake \
+ -DBOOTSTRAP_LLVM_PARALLEL_LINK_JOBS=1 \
+ -DBOOTSTRAP_CPACK_PACKAGE_FILE_NAME="${{ needs.prepare.outputs.release-binary-basename }}" \
+ -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+ ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build
+ # There is a race condition on the MacOS builders and this command is here
+ # to help debug that when it happens.
+ ls -ltr ${{ steps.setup-stage.outputs.build-prefix }}/build
+
+ - name: Save Stage
+ uses: ./.github/workflows/release-binaries-save-stage
with:
- name: stage1-source
- path: llvm-project.tar.zst
- retention-days: 2
+ build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
- - name: Upload Stage 1 Build Dir
- uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
- with:
- name: stage1-build
- path: build.tar.zst
- retention-days: 2
-
- build-stage2-linux:
- name: "Build Stage 2 Linux"
+ build-stage2:
+ name: "Build Stage 2"
needs:
- prepare
- - build-stage1-linux
- runs-on: ubuntu-22.04
+ - build-stage1
if: github.repository == 'llvm/llvm-project'
+ runs-on: ${{ inputs.runs-on }}
steps:
- - name: Install Ninja
- uses: llvm/actions/install-ninja at 22e9f909d35b50bd1181709564bfe816eaeaae81 # main
-
- - name: Download Stage 1 Artifacts
- uses: actions/download-artifact at 6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+ - name: Checkout Actions
+ uses: actions/checkout at v4
with:
- pattern: stage1-*
- merge-multiple: true
-
- - name: Unpack Artifacts
- run: |
- tar --zstd -xf llvm-project.tar.zst
- rm llvm-project.tar.zst
- sudo chown $USER:$USER /mnt/
- tar --zstd -C /mnt -xf build.tar.zst
- rm build.tar.zst
+ ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
+ sparse-checkout: |
+ .github/workflows/
+ sparse-checkout-cone-mode: false
+ path: workflows
+ - name: Setup Stage
+ id: setup-stage
+ uses: ./workflows/.github/workflows/release-binaries-setup-stage
+ with:
+ previous-artifact: build-stage1
- name: Build Stage 2
# Re-enable once PGO builds are supported.
- if: false
- run: |
- ninja -C /mnt/build stage2-instrumented
-
- # We need to create an archive of the build directory, because it has too
- # many files to upload.
- - name: Save Build and Source Directories
+ if: needs.prepare.outputs.enable-pgo == 'true'
+ shell: bash
run: |
- tar -c . | zstd -T0 -c > llvm-project.tar.zst
- tar -C /mnt/ -c build/ | zstd -T0 -c > build.tar.zst
+ ninja -C ${{ steps.setup-stage.outputs.build-prefix}}/build stage2-instrumented
- - name: Upload Stage 2 Source
- uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ - name: Save Stage
+ uses: ./workflows/.github/workflows/release-binaries-save-stage
with:
- name: stage2-source
- path: ${{ github.workspace }}/llvm-project.tar.zst
- retention-days: 2
+ build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
- - name: Upload Stage 2 Build Dir
- uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ build-stage3-clang:
+ name: "Build Stage 3 LLVM/Clang"
+ needs:
+ - prepare
+ - build-stage2
+ if: github.repository == 'llvm/llvm-project'
+ runs-on: ${{ inputs.runs-on }}
+ steps:
+ - name: Checkout Actions
+ uses: actions/checkout at v4
with:
- name: stage2-build
- path: ${{ github.workspace }}/build.tar.zst
- retention-days: 2
+ ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
+ sparse-checkout: |
+ .github/workflows/
+ sparse-checkout-cone-mode: false
+ path: workflows
+ - name: Setup Stage
+ id: setup-stage
+ uses: ./workflows/.github/workflows/release-binaries-setup-stage
+ with:
+ previous-artifact: build-stage2
+ - name: Build LLVM/Clang
+ shell: bash
+ run: |
+ # There is a race condition on the MacOS builders and this command is here
+ # to help debug that when it happens.
+ ls -ltr ${{ steps.setup-stage.outputs.build-prefix }}/build
+ ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-clang
+ # Build some of the larger binaries here too.
+ ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/ \
+ clang-scan-deps \
+ modularize clangd \
+ clangd-indexer \
+ clang-check \
+ ${{ (runner.os == 'Linux' && 'clangd-fuzzer') || '' }} \
+ clang-tidy \
+ llc \
+ lli \
+ llvm-exegesis \
+ llvm-opt-fuzzer \
+ llvm-reduce \
+ llvm-lto \
+ dsymutil
+
+ - name: Save Stage
+ uses: ./workflows/.github/workflows/release-binaries-save-stage
+ with:
+ build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
- build-stage3-linux:
- name: "Build Stage 3 Linux"
+ build-stage3-flang:
+ name: "Build Stage 3 Flang/MLIR/Bolt"
needs:
- prepare
- - build-stage2-linux
- outputs:
- filename: ${{ steps.package-info.outputs.release-filename }}
- runs-on: ubuntu-22.04-16x64
- if: github.repository == 'llvm/llvm-project'
+ - build-stage3-clang
+ runs-on: ${{ inputs.runs-on }}
steps:
- - name: Install Ninja
- uses: llvm/actions/install-ninja at 22e9f909d35b50bd1181709564bfe816eaeaae81 # main
-
- - name: 'Download artifact'
- uses: actions/download-artifact at 6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+ - name: Checkout Actions
+ uses: actions/checkout at v4
with:
- pattern: stage2-*
- merge-multiple: true
+ ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
+ sparse-checkout: |
+ .github/workflows/
+ sparse-checkout-cone-mode: false
+ path: workflows
+ - name: Setup Stage
+ id: setup-stage
+ uses: ./workflows/.github/workflows/release-binaries-setup-stage
+ with:
+ previous-artifact: build-stage3-clang
- - name: Unpack Artifact
+ - name: Build Flang / MLIR / Bolt
+ shell: bash
run: |
- tar --zstd -xf llvm-project.tar.zst
- rm llvm-project.tar.zst
- sudo chown $USER:$USER /mnt/
- tar --zstd -C /mnt -xf build.tar.zst
- rm build.tar.zst
+ # Build some of the mlir tools that take a long time to link
+ if [ "${{ needs.prepare.outputs.build-flang }}" = "true" ]; then
+ ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/ -j2 flang-new bbc
+ fi
+ ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/ \
+ mlir-bytecode-parser-fuzzer \
+ mlir-cpu-runner \
+ mlir-lsp-server \
+ mlir-opt \
+ mlir-query \
+ mlir-reduce \
+ mlir-text-parser-fuzzer \
+ mlir-translate \
+ mlir-transform-opt \
+ mlir-cat \
+ mlir-minimal-opt \
+ mlir-minimal-opt-canonicalize \
+ mlir-pdll-lsp-server \
+ llvm-bolt \
+ llvm-bolt-heatmap
+
+ - name: Save Stage
+ uses: ./workflows/.github/workflows/release-binaries-save-stage
+ with:
+ build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
- - name: Build Release Package
- run: |
- ninja -C /mnt/build stage2-package
+ build-stage3-all:
+ name: "Build Stage 3"
+ needs:
+ - prepare
+ - build-stage3-flang
+ runs-on: ${{ inputs.runs-on }}
+ steps:
+ - name: Checkout Actions
+ uses: actions/checkout at v4
+ with:
+ ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
+ sparse-checkout: |
+ .github/workflows/
+ sparse-checkout-cone-mode: false
+ path: workflows
+ - name: Setup Stage
+ id: setup-stage
+ uses: ./workflows/.github/workflows/release-binaries-setup-stage
+ with:
+ previous-artifact: build-stage3-flang
- - id: package-info
+ - name: Build Release Package
+ shell: bash
run: |
- filename="LLVM-${{ needs.prepare.outputs.release-version }}-Linux.tar.xz"
- echo "filename=$filename" >> $GITHUB_OUTPUT
- echo "path=/mnt/build/tools/clang/stage2-bins/$filename" >> $GITHUB_OUTPUT
+ which cmake
+ ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-package
+ # Copy Release artifact to the workspace so it is easier to upload.
+ # This is necessary, because on Windows, the build-prefix path can
+ # only be used on bash steps, because it uses the form of /d/files/
+ # and other steps expect D:\files.
+ mv ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/${{ needs.prepare.outputs.release-binary-filename }} .
- uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
- if: always()
with:
- name: release-binary
- path: ${{ steps.package-info.outputs.path }}
+ name: ${{ runner.os }}-${{ runner.arch }}-release-binary
+ # Due to path differences on Windows when running in bash vs running on node,
+ # we need to search for files in the current workspace.
+ path: |
+ ${{ needs.prepare.outputs.release-binary-filename }}
# Clean up some build files to reduce size of artifact.
- name: Clean Up Build Directory
+ shell: bash
run: |
- find /mnt/build -iname ${{ steps.package-info.outputs.filename }} -delete
-
- # We need to create an archive of the build directory, because it has too
- # many files to upload.
- - name: Save Build and Source Directories
- run: |
- tar -c . | zstd -T0 -c > llvm-project.tar.zst
- tar -C /mnt/ -c build/ | zstd -T0 -c > build.tar.zst
-
- - name: Upload Stage 3 Source
- uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
- with:
- name: stage3-source
- path: llvm-project.tar.zst
- retention-days: 2
+ find ${{ steps.setup-stage.outputs.build-prefix }}/build -iname ${{ needs.prepare.outputs.release-binary-filename }} -delete
+ rm -Rf ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/_CPack_Packages
- - name: Upload Stage 3 Build Dir
- uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ - name: Save Stage
+ uses: ./workflows/.github/workflows/release-binaries-save-stage
with:
- name: stage3-build
- path: build.tar.zst
- retention-days: 2
+ build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
- upload-release-binaries-linux:
- name: "Upload Linux Release Binaries"
+ upload-release-binaries:
+ name: "Upload Release Binaries"
needs:
- prepare
- - build-stage3-linux
- if : ${{ needs.prepare.outputs.upload == 'true' }}
+ - build-stage3-all
+ if: >-
+ always() &&
+ github.event_name != 'pull_request' &&
+ needs.prepare.outputs.upload == 'true'
runs-on: ubuntu-22.04
permissions:
contents: write # For release uploads
+ id-token: write # For artifact attestations
+ attestations: write # For artifact attestations
steps:
- name: 'Download artifact'
uses: actions/download-artifact at 6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
with:
- name: release-binary
+ pattern: '*-release-binary'
+ merge-multiple: true
+
+ - name: Attest Build Provenance
+ id: provenance
+ uses: actions/attest-build-provenance at 897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0
+ with:
+ subject-path: ${{ needs.prepare.outputs.release-binary-filename }}
+
+ - name: Rename attestation file
+ run:
+ mv ${{ steps.provenance.outputs.bundle-path }} ${{ needs.prepare.outputs.release-binary-filename }}.jsonl
+
+ - name: Upload Build Provenance
+ uses: actions/upload-artifact at 65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3
+ with:
+ name: ${{ runner.os }}-${{ runner.arch }}-release-binary-attestation
+ path: ${{ needs.prepare.outputs.release-binary-filename }}.jsonl
- name: Upload Release
+ shell: bash
run: |
sudo apt install python3-github
./llvm-project/llvm/utils/release/github-upload-release.py \
--token ${{ github.token }} \
--release ${{ needs.prepare.outputs.release-version }} \
upload \
- --files ${{ needs.build-stage3-linux.outputs.release-filename }}
-
+ --files ${{ needs.prepare.outputs.release-binary-filename }}*
- test-stage3-linux:
- name: "Test Stage 3 Linux"
+ test-stage3:
+ name: "Test Stage 3"
needs:
- prepare
- - build-stage3-linux
- runs-on: ubuntu-22.04
- if: github.repository == 'llvm/llvm-project'
+ - build-stage3-all
+ if: >-
+ github.repository == 'llvm/llvm-project'
+ runs-on: ${{ inputs.runs-on }}
steps:
- - name: Install Ninja
- uses: llvm/actions/install-ninja at 22e9f909d35b50bd1181709564bfe816eaeaae81 # main
-
- - name: 'Download artifact'
- uses: actions/download-artifact at 6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+ - name: Checkout Actions
+ uses: actions/checkout at v4
with:
- pattern: stage3-*
- merge-multiple: true
-
- - name: Unpack Artifact
- run: |
- tar --zstd -xf llvm-project.tar.zst
- rm llvm-project.tar.zst
- sudo chown $USER:$USER /mnt/
- tar --zstd -C /mnt -xf build.tar.zst
- rm build.tar.zst
+ ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
+ sparse-checkout: |
+ .github/workflows/
+ sparse-checkout-cone-mode: false
+ path: workflows
+ - name: Setup Stage
+ id: setup-stage
+ uses: ./workflows/.github/workflows/release-binaries-setup-stage
+ with:
+ previous-artifact: build-stage3-all
- name: Run Tests
+ shell: bash
run: |
- ninja -C /mnt/build stage2-check-all
+ ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-check-all
diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml
index 2ed56dace1d4c..7dd4c306671b7 100644
--- a/.github/workflows/release-tasks.yml
+++ b/.github/workflows/release-tasks.yml
@@ -81,10 +81,20 @@ jobs:
needs:
- validate-tag
- release-create
+ strategy:
+ fail-fast: false
+ matrix:
+ runs-on:
+ - ubuntu-22.04
+ - windows-2022
+ - macos-13
+ - macos-14
+
uses: ./.github/workflows/release-binaries.yml
with:
release-version: ${{ needs.validate-tag.outputs.release-version }}
upload: true
+ runs-on: ${{ matrix.runs-on }}
release-sources:
name: Package Release Sources
diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake
index 9e6feb479d45f..e5161dd9a27b9 100644
--- a/clang/cmake/caches/Release.cmake
+++ b/clang/cmake/caches/Release.cmake
@@ -29,9 +29,13 @@ endfunction()
# cache file to CMake via -C. e.g.
#
# cmake -D LLVM_RELEASE_ENABLE_PGO=ON -C Release.cmake
+set (DEFAULT_RUNTIMES "compiler-rt;libcxx")
+if (NOT WIN32)
+ list(APPEND DEFAULT_RUNTIMES "libcxxabi" "libunwind")
+endif()
set(LLVM_RELEASE_ENABLE_LTO THIN CACHE STRING "")
set(LLVM_RELEASE_ENABLE_PGO ON CACHE BOOL "")
-set(LLVM_RELEASE_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
+set(LLVM_RELEASE_ENABLE_RUNTIMES ${DEFAULT_RUNTIMES} CACHE STRING "")
set(LLVM_RELEASE_ENABLE_PROJECTS "clang;lld;lldb;clang-tools-extra;bolt;polly;mlir;flang" CACHE STRING "")
# Note we don't need to add install here, since it is one of the pre-defined
# steps.
>From b78a2dba50cb99033000fefd9967e0f9ac127559 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 26 Jul 2024 12:36:40 -0700
Subject: [PATCH 3/6] workflows: Remove left over debugging step from
release-binaries job
(cherry picked from commit 18dee70168bcd7259daade4c86462ba859e7bed5)
---
.github/workflows/release-binaries.yml | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 5f939ba5bbe2f..0a800fd006e9d 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -152,17 +152,6 @@ jobs:
uses: actions/checkout at b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ needs.prepare.outputs.ref }}
- - name: Debug - Move actions
- if: github.event_name != 'pull_request'
- shell: bash
- run: |
- cd .github/workflows
- for d in release-binaries-setup-stage release-binaries-save-stage; do
- mkdir $d
- pushd $d
- curl -O -L https://raw.githubusercontent.com/tstellar/llvm-project/main/.github/workflows/$d/action.yml
- popd
- done
- name: Setup sccache
uses: hendrikmuhs/ccache-action at ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9
>From 03a191ac6e7d0db5bbc35d9a20265a2de237cc04 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 26 Jul 2024 14:46:32 -0700
Subject: [PATCH 4/6] workflows/release-binaries: Always pull composite actions
from main branch (#100805)
If we pull from the release tag, then if there is a bug in one of the
actions on the release tag, then we can never do a build for that tag.
Pulling from main will allows us to fix bugs in the actions we use to
build the releases.
(cherry picked from commit b0860b20878d2c84fc3ce56ea608c5186872faa2)
---
.github/workflows/release-binaries.yml | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 0a800fd006e9d..38b2ba7ba12dc 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -148,6 +148,20 @@ jobs:
if: github.repository == 'llvm/llvm-project'
runs-on: ${{ inputs.runs-on }}
steps:
+
+ - name: Checkout Actions
+ uses: actions/checkout@@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+ with:
+ ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
+ sparse-checkout: |
+ .github/workflows/
+ sparse-checkout-cone-mode: false
+ path: workflows
+
+ - name: Setup Stage
+ id: setup-stage
+ uses: ./workflows/.github/workflows/release-binaries-setup-stage
+
- name: Checkout LLVM
uses: actions/checkout at b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
@@ -161,10 +175,6 @@ jobs:
key: sccache-${{ runner.os }}-${{ runner.arch }}-release
variant: sccache
- - name: Setup Stage
- id: setup-stage
- uses: ./.github/workflows/release-binaries-setup-stage
-
- name: Build Stage 1 Clang
id: build
shell: bash
@@ -184,7 +194,7 @@ jobs:
ls -ltr ${{ steps.setup-stage.outputs.build-prefix }}/build
- name: Save Stage
- uses: ./.github/workflows/release-binaries-save-stage
+ uses: ./workflows/.github/workflows/release-binaries-save-stage
with:
build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
>From df34f39da3cf92c2de3b8daf6b47d229099912f6 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 26 Jul 2024 14:51:47 -0700
Subject: [PATCH 5/6] workflow/release-binaries: Fix typo
Introduced in b0860b20878d2c84fc3ce56ea608c5186872faa2.
(cherry picked from commit d41f565318e2a414acfd7eec1cfb2fbf515391ba)
---
.github/workflows/release-binaries.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 38b2ba7ba12dc..f7ed88ff03add 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -150,7 +150,7 @@ jobs:
steps:
- name: Checkout Actions
- uses: actions/checkout@@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+ uses: actions/checkout at b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
sparse-checkout: |
>From a73b708c9dfda3c0205b3309170d06b7acbbed00 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Tue, 30 Jul 2024 18:54:20 -0700
Subject: [PATCH 6/6] workflows/release-binaries: Fetch composite actions
outside of default workspace (#100845)
Otherwise, the checkout step will override them.
(cherry picked from commit 41003ff3fe344dee5c963d462a4bc6d528811d86)
---
.github/workflows/release-binaries.yml | 25 +++++++++++++++++++++----
1 file changed, 21 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index f7ed88ff03add..b1b046dbad5f8 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -156,17 +156,34 @@ jobs:
sparse-checkout: |
.github/workflows/
sparse-checkout-cone-mode: false
+ # Check out outside of working directory so the source checkout doesn't
+ # remove it.
path: workflows
- - name: Setup Stage
- id: setup-stage
- uses: ./workflows/.github/workflows/release-binaries-setup-stage
+ # actions/checkout does not support paths outside of the GITHUB_WORKSPACE.
+ # Also, anything that we put inside of GITHUB_WORKSPACE will be overwritten
+ # by future actions/checkout steps. Therefore, in order to checkout the
+ # latest actions from main, we need to first checkout out the actions inside of
+ # GITHUB_WORKSPACE (see previous step), then use actions/checkout to checkout
+ # the code being built and the move the actions from main back into GITHUB_WORKSPACE,
+ # becasue the uses on composite actions only reads workflows from inside GITHUB_WORKSPACE.
+ - shell: bash
+ run: mv workflows ../workflows-main
- name: Checkout LLVM
uses: actions/checkout at b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ needs.prepare.outputs.ref }}
+ - name: Copy main workflows
+ shell: bash
+ run: |
+ mv ../workflows-main .
+
+ - name: Setup Stage
+ id: setup-stage
+ uses: ./workflows-main/.github/workflows/release-binaries-setup-stage
+
- name: Setup sccache
uses: hendrikmuhs/ccache-action at ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9
with:
@@ -194,7 +211,7 @@ jobs:
ls -ltr ${{ steps.setup-stage.outputs.build-prefix }}/build
- name: Save Stage
- uses: ./workflows/.github/workflows/release-binaries-save-stage
+ uses: ./workflows-main/.github/workflows/release-binaries-save-stage
with:
build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
More information about the llvm-branch-commits
mailing list