[llvm] [workflows] Port buildkite Windows config to GitHub actions (PR #82093)
Tom Stellard via llvm-commits
llvm-commits at lists.llvm.org
Sat Apr 27 16:15:24 PDT 2024
https://github.com/tstellar updated https://github.com/llvm/llvm-project/pull/82093
>From 3093657b77363c0f1066cb9aaf76623c2ef32286 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 16 Feb 2024 21:34:02 +0000
Subject: [PATCH 1/2] [workflows] Port buildkite Windows config to GitHub
actions
This reuses most of the generate-buildkite-pipeline-premerge script
which determines which projects to build and which check targets to
use.
This new workflow only tests clang, llvm, and lld due to resource
contraints on the GitHub runners.
---
.../compute-projects-to-test/action.yml | 21 ++
.../compute-projects-to-test.sh | 220 ++++++++++++++++++
.github/workflows/precommit-windows.yml | 61 +++++
3 files changed, 302 insertions(+)
create mode 100644 .github/workflows/compute-projects-to-test/action.yml
create mode 100755 .github/workflows/compute-projects-to-test/compute-projects-to-test.sh
create mode 100644 .github/workflows/precommit-windows.yml
diff --git a/.github/workflows/compute-projects-to-test/action.yml b/.github/workflows/compute-projects-to-test/action.yml
new file mode 100644
index 00000000000000..37df06c8c301c5
--- /dev/null
+++ b/.github/workflows/compute-projects-to-test/action.yml
@@ -0,0 +1,21 @@
+name: 'Compute Projects To Test'
+inputs:
+ projects:
+ required: false
+ type: 'string'
+
+outputs:
+ check-targets:
+ description: "A space delimited list of check-targets to pass to ninja."
+ value: ${{ steps.compute-projects.outputs.check-targets }}
+
+ projects:
+ description: "A semi-colon delimited list of projects to pass to -DLLVM_ENABLE_PROJECTS."
+ value: ${{ steps.compute-projects.outputs.projects }}
+
+runs:
+ using: "composite"
+ steps:
+ - id: compute-projects
+ run: .github/workflows/compute-projects-to-test/compute-projects-to-test.sh ${{ inputs.projects }}
+ shell: bash
diff --git a/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh
new file mode 100755
index 00000000000000..807142668f618b
--- /dev/null
+++ b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh
@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+#
+# This file generates a Buildkite pipeline that triggers the various CI jobs for
+# the LLVM project during pre-commit CI.
+#
+# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
+#
+# As this outputs a yaml file, it's possible to log messages to stderr or
+# prefix with "#".
+
+
+set -eu
+set -o pipefail
+
+# Environment variables script works with:
+
+# Set by GitHub
+: ${GITHUB_OUTPUT:=}
+: ${RUNNER_OS:=}
+
+# Allow users to specify which projects to build.
+all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
+if [ "$#" -ne 0 ]; then
+ wanted_projects="${@}"
+else
+ wanted_projects="${all_projects}"
+fi
+
+# List of files affected by this commit
+: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1...HEAD)}
+
+echo "Files modified:" >&2
+echo "$MODIFIED_FILES" >&2
+modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
+echo "Directories modified:" >&2
+echo "$modified_dirs" >&2
+echo "wanted_projects: $wanted_projects"
+
+function remove-unwanted-projects() {
+ projects=${@}
+ for project in ${projects}; do
+ if echo "$wanted_projects" | tr ' ' '\n' | grep -q -E "^${project}$"; then
+ echo "${project}"
+ fi
+ done
+}
+
+function compute-projects-to-test() {
+ projects=${@}
+ for project in ${projects}; do
+ echo "${project}"
+ case ${project} in
+ lld)
+ for p in bolt cross-project-tests; do
+ echo $p
+ done
+ ;;
+ llvm)
+ for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do
+ echo $p
+ done
+ ;;
+ clang)
+ for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do
+ echo $p
+ done
+ ;;
+ clang-tools-extra)
+ echo libc
+ ;;
+ mlir)
+ echo flang
+ ;;
+ *)
+ # Nothing to do
+ ;;
+ esac
+ done
+}
+
+function add-dependencies() {
+ projects=${@}
+ for project in ${projects}; do
+ echo "${project}"
+ case ${project} in
+ bolt)
+ for p in lld llvm; do
+ echo $p
+ done
+ ;;
+ cross-project-tests)
+ for p in lld clang; do
+ echo $p
+ done
+ ;;
+ clang-tools-extra)
+ for p in llvm clang; do
+ echo $p
+ done
+ ;;
+ compiler-rt|libc|openmp)
+ echo clang lld
+ ;;
+ flang|lldb)
+ for p in llvm clang; do
+ echo $p
+ done
+ ;;
+ lld|mlir|polly)
+ echo llvm
+ ;;
+ *)
+ # Nothing to do
+ ;;
+ esac
+ done
+}
+
+function exclude-linux() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ cross-project-tests) ;; # tests failing
+ lldb) ;; # tests failing
+ openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410
+ *)
+ echo "${project}"
+ ;;
+ esac
+ done
+}
+
+function exclude-windows() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ cross-project-tests) ;; # tests failing
+ compiler-rt) ;; # tests taking too long
+ openmp) ;; # TODO: having trouble with the Perl installation
+ libc) ;; # no Windows support
+ lldb) ;; # tests failing
+ bolt) ;; # tests are not supported yet
+ *)
+ echo "${project}"
+ ;;
+ esac
+ done
+}
+
+# Prints only projects that are both present in $modified_dirs and the passed
+# list.
+function keep-modified-projects() {
+ projects=${@}
+ for project in ${projects}; do
+ if echo "$modified_dirs" | grep -q -E "^${project}$"; then
+ echo "${project}"
+ fi
+ done
+}
+
+function check-targets() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ clang-tools-extra)
+ echo "check-clang-tools"
+ ;;
+ compiler-rt)
+ echo "check-all"
+ ;;
+ cross-project-tests)
+ echo "check-cross-project"
+ ;;
+ lldb)
+ echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
+ ;;
+ pstl)
+ echo "check-all"
+ ;;
+ libclc)
+ echo "check-all"
+ ;;
+ *)
+ echo "check-${project}"
+ ;;
+ esac
+ done
+}
+
+# Generic pipeline for projects that have not defined custom steps.
+#
+# Individual projects should instead define the pre-commit CI tests that suits their
+# needs while letting them run on the infrastructure provided by LLVM.
+
+# Figure out which projects need to be built on each platform
+modified_projects="$(keep-modified-projects ${all_projects})"
+echo "modified_projects: $modified_projects"
+
+if [ "${RUNNER_OS}" = "Linux" ]; then
+ projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
+elif [ "${RUNNER_OS}" = "Windows" ]; then
+ projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
+else
+ echo "Unknown runner OS: $RUNNER_OS"
+fi
+check_targets=$(check-targets $(remove-unwanted-projects ${projects_to_test}) | sort | uniq)
+projects=$(remove-unwanted-projects $(add-dependencies ${projects_to_test}) | sort | uniq)
+
+echo "check-targets=$(echo ${check_targets} | tr ' ' ' ')" >> $GITHUB_OUTPUT
+echo "projects=$(echo ${projects} | tr ' ' ';')" >> $GITHUB_OUTPUT
+
+cat $GITHUB_OUTPUT
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
new file mode 100644
index 00000000000000..4864ab72241dde
--- /dev/null
+++ b/.github/workflows/precommit-windows.yml
@@ -0,0 +1,61 @@
+name: "Windows Precommit Tests"
+
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ branches:
+ - main
+
+concurrency:
+ # Skip intermediate builds: always.
+ # Cancel intermediate builds: only if it is a pull request build.
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+ build-llvm-windows:
+ name: "Build and test LLVM (Windows)"
+ runs-on: windows-2022
+ steps:
+ - name: Setup Windows
+ uses: llvm/actions/setup-windows at main
+ with:
+ arch: amd64
+ - name: Fetch LLVM sources
+ uses: actions/checkout at v4
+ with:
+ fetch-depth: 2
+ - name: Setup ccache
+ uses: hendrikmuhs/ccache-action at v1
+ with:
+ max-size: 500M
+ variant: sccache
+ key: precommit-windows
+ - name: Compute projects to test
+ id: compute-projects
+ uses: ./.github/workflows/compute-projects-to-test
+ with:
+ projects: clang llvm lld
+
+ - name: Configure LLVM
+ shell: bash
+ if: ${{ steps.compute-projects.outputs.check-targets }}
+ run: |
+ cmake -B build -GNinja \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_ENABLE_PROJECTS="${{ steps.compute-projects.outputs.projects }}" \
+ -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+ -DLLVM_ENABLE_ASSERTIONS=ON \
+ -DLLVM_LIT_ARGS="-v --no-progress-bar" \
+ -S llvm
+ - name: Build LLVM
+ if: ${{ steps.compute-projects.outputs.check-targets }}
+ run: |
+ ninja -C build test-depends
+ - name: Check LLVM
+ if: ${{ steps.compute-projects.outputs.check-targets }}
+ run: |
+ ninja -C build "${{ steps.compute-projects.outputs.check-targets }}"
>From 673c0788673e27ca43dcc869e0506adc1ec87a30 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 05:04:31 +0000
Subject: [PATCH 2/2] Workflow improvements:
* Split the build into 3 jobs to avoid timeouts.
* Maintiain an scacche cache that is unique for each PR.
---
.../workflows/pr-sccache-restore/action.yml | 44 +++++
.github/workflows/pr-sccache-save/action.yml | 38 ++++
.github/workflows/precommit-windows.yml | 171 ++++++++++++++++--
3 files changed, 236 insertions(+), 17 deletions(-)
create mode 100644 .github/workflows/pr-sccache-restore/action.yml
create mode 100644 .github/workflows/pr-sccache-save/action.yml
diff --git a/.github/workflows/pr-sccache-restore/action.yml b/.github/workflows/pr-sccache-restore/action.yml
new file mode 100644
index 00000000000000..9b620c7bdcbbe6
--- /dev/null
+++ b/.github/workflows/pr-sccache-restore/action.yml
@@ -0,0 +1,44 @@
+name: PR sccache restore
+
+runs:
+ using: "composite"
+ steps:
+ - uses: actions/github-script at 60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+ id: artifact-url
+ with:
+ script: |
+ const data = await github.rest.actions.listArtifactsForRepo({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ name: 'sccache-pr' + context.issue.number
+ })
+
+ console.log(data.data.artifacts)
+
+ for (artifact of data.data.artifacts) {
+ if (artifact.expired) {
+ continue;
+ }
+ console.log(artifact)
+ const url = await github.rest.actions.downloadArtifact({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ artifact_id: artifact.id,
+ archive_format: "zip"
+ })
+ console.log(url.url)
+ return url.url
+ }
+ console.log("Could not find previous sccache for this PR.")
+ return ""
+
+ - shell: bash
+ if: steps.artifact-url.outputs.result != ''
+ run: |
+ curl -L -o sccache-pr${{ github.event.number }}.zip ${{ steps.artifact-url.outputs.result }}
+ # Is this the best way to clear the cache?
+ rm -Rf .sccache/
+ unzip -d .sccache sccache-pr${{ github.event.number}}.zip
+ ls -ltr
+
+
diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml
new file mode 100644
index 00000000000000..43f0cb29115b87
--- /dev/null
+++ b/.github/workflows/pr-sccache-save/action.yml
@@ -0,0 +1,38 @@
+name: PR sccache save
+
+runs:
+ using: "composite"
+ steps:
+ - uses: actions/github-script at 60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+ with:
+ script: |
+ const data = await github.rest.actions.listWorkflowRunArtifacts({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ name: 'sccache-pr' + context.issue.number
+ })
+
+ console.log(data.data.artifacts)
+ if (data.data.artifacts.length == 0) {
+ return '';
+ }
+ console.log(data.data.artifacts[0])
+ const artifact_id = data.data.artifacts[0].id
+
+ // Delete the exisitng artifact so we can upload a new one with the same name.
+ github.rest.actions.deleteArtifact({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ artifact_id: artifact_id
+ })
+
+ - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ with:
+ name: 'sccache-pr${{ github.event.number }}'
+ path: .sccache
+ retention-days: 7
+
+ - shell: bash
+ run: |
+ sccache --show-stats
+
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index 4864ab72241dde..ddcaa656215b3c 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -15,47 +15,184 @@ concurrency:
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
jobs:
- build-llvm-windows:
- name: "Build and test LLVM (Windows)"
+ compute-projects:
+ name: "Compute Projects to Test"
+ runs-on: ubuntu-22.04
+ outputs:
+ projects: ${{ steps.vars.outputs.projects }}
+ check-targets: ${{ steps.vars.outputs.check-targets }}
+ test-build: ${{ steps.vars.outputs.check-targets != '' }}
+ steps:
+ - name: Fetch LLVM sources
+ uses: actions/checkout at v4
+ with:
+ fetch-depth: 2
+
+ - name: Compute projects to test
+ id: vars
+ uses: ./.github/workflows/compute-projects-to-test
+
+ build-windows-part1:
+ name: "Build LLVM/Clang"
runs-on: windows-2022
+ needs:
+ - compute-projects
+ if: ${{ needs.compute-projects.outputs.test-build == 'true' }}
steps:
- name: Setup Windows
uses: llvm/actions/setup-windows at main
with:
arch: amd64
+
- name: Fetch LLVM sources
uses: actions/checkout at v4
- with:
- fetch-depth: 2
- - name: Setup ccache
+
+ - name: Setup sccache
uses: hendrikmuhs/ccache-action at v1
with:
- max-size: 500M
+ max-size: 2G
variant: sccache
key: precommit-windows
+
+ - name: Restore sccache from previous PR run
+ uses: ./.github/workflows/pr-sccache-restore
+
- name: Compute projects to test
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
- with:
- projects: clang llvm lld
- name: Configure LLVM
shell: bash
- if: ${{ steps.compute-projects.outputs.check-targets }}
run: |
cmake -B build -GNinja \
-DCMAKE_BUILD_TYPE=Release \
- -DLLVM_ENABLE_PROJECTS="${{ steps.compute-projects.outputs.projects }}" \
- -DCMAKE_C_COMPILER_LAUNCHER=sccache \
- -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+ -DLLVM_ENABLE_PROJECTS="${{ needs.compute-projects.outputs.projects }}" \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_LIT_ARGS="-v --no-progress-bar" \
+ -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-S llvm
- - name: Build LLVM
- if: ${{ steps.compute-projects.outputs.check-targets }}
+
+ - name: Build
+ shell: bash
+ run: |
+ targets="llc"
+ if echo "${{ needs.compute-projects.outputs.check-targets }}" | grep clang; then
+ targets="$targets clang"
+ fi
+ ninja -C build $targets
+
+ - name: Save sccache for next PR run
+ if: always()
+ uses: ./.github/workflows/pr-sccache-save
+
+ - name: Package Build Directory
+ shell: bash
run: |
- ninja -C build test-depends
+ tar -c . | zstd -T0 -c > ../llvm-project.tar.zst
+ mv ../llvm-project.tar.zst .
+
+ - name: Upload Build
+ uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ with:
+ name: part1
+ path: llvm-project.tar.zst
+ retention-days: 2
+
+ build-windows-part2:
+ name: "Build All"
+ needs:
+ - build-windows-part1
+ runs-on: windows-2022
+ steps:
+ - name: Setup Windows
+ uses: llvm/actions/setup-windows at main
+ with:
+ arch: amd64
+
+ - name: Setup ccache
+ uses: hendrikmuhs/ccache-action at v1
+ with:
+ max-size: 2G
+ variant: sccache
+ key: precommit-windows
+ # The llvm-project.zip archive contains the sccache directory from
+ # part 1, so we don't need to restore the cache.
+ restore: false
+
+ - name: Download Artifact
+ uses: actions/download-artifact at 6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+ with:
+ pattern: part1
+
+ - name: Unpack Artifact
+ shell: bash
+ run: |
+ tar --zstd -xf llvm-project.tar.zst
+ rm llvm-project.tar.zst
+
+ - name: Build
+ shell: bash
+ run: |
+ ls
+ ninja -C build
+
+ - name: Save sccache for next PR run
+ if: always()
+ uses: ./.github/workflows/pr-sccache-save
+
+ - name: Package Build Directory
+ shell: bash
+ run: |
+ tar -c . | zstd -T0 -c > ../llvm-project.tar.zst
+ mv ../llvm-project.tar.zst .
+
+ - name: Upload Build
+ uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+ with:
+ name: part2
+ path: llvm-project.tar.zst
+ retention-days: 2
+
+ test-windows:
+ name: "Test"
+ needs:
+ - compute-projects
+ - build-windows-part2
+ runs-on: windows-2022
+ steps:
+ - name: Setup Windows
+ uses: llvm/actions/setup-windows at main
+ with:
+ arch: amd64
+
+ - name: Setup ccache
+ uses: hendrikmuhs/ccache-action at v1
+ with:
+ max-size: 2G
+ variant: sccache
+ key: precommit-windows
+ # The llvm-project.zip archive contains the sccache directory from
+ # part 1, so we don't need to restore the cache.
+ restore: false
+
+ - name: Download Artifact
+ uses: actions/download-artifact at 6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+ with:
+ pattern: part2
+ merge-multiple: true
+
+ - name: Unpack Artifact
+ shell: bash
+ run: |
+ tar --zstd -xf llvm-project.tar.zst
+ rm llvm-project.tar.zst
+
- name: Check LLVM
- if: ${{ steps.compute-projects.outputs.check-targets }}
+ if: ${{ needs.compute-projects.outputs.check-targets }}
run: |
- ninja -C build "${{ steps.compute-projects.outputs.check-targets }}"
+ ninja -C build ${{ needs.compute-projects.outputs.check-targets }}
+
+ - name: Save sccache for next PR run
+ if: always()
+ uses: ./.github/workflows/pr-sccache-save
More information about the llvm-commits
mailing list