[llvm] Buildkite gh mlir (PR #82139)
Tom Stellard via llvm-commits
llvm-commits at lists.llvm.org
Sat Feb 17 16:03:13 PST 2024
https://github.com/tstellar created https://github.com/llvm/llvm-project/pull/82139
None
>From 3093657b77363c0f1066cb9aaf76623c2ef32286 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 16 Feb 2024 21:34:02 +0000
Subject: [PATCH 01/11] [workflows] Port buildkite Windows config to GitHub
actions
This reuses most of the generate-buildkite-pipeline-premerge script
which determines which projects to build and which check targets to
use.
This new workflow only tests clang, llvm, and lld due to resource
contraints on the GitHub runners.
---
.../compute-projects-to-test/action.yml | 21 ++
.../compute-projects-to-test.sh | 220 ++++++++++++++++++
.github/workflows/precommit-windows.yml | 61 +++++
3 files changed, 302 insertions(+)
create mode 100644 .github/workflows/compute-projects-to-test/action.yml
create mode 100755 .github/workflows/compute-projects-to-test/compute-projects-to-test.sh
create mode 100644 .github/workflows/precommit-windows.yml
diff --git a/.github/workflows/compute-projects-to-test/action.yml b/.github/workflows/compute-projects-to-test/action.yml
new file mode 100644
index 00000000000000..37df06c8c301c5
--- /dev/null
+++ b/.github/workflows/compute-projects-to-test/action.yml
@@ -0,0 +1,21 @@
+name: 'Compute Projects To Test'
+inputs:
+ projects:
+ required: false
+ type: 'string'
+
+outputs:
+ check-targets:
+ description: "A space delimited list of check-targets to pass to ninja."
+ value: ${{ steps.compute-projects.outputs.check-targets }}
+
+ projects:
+ description: "A semi-colon delimited list of projects to pass to -DLLVM_ENABLE_PROJECTS."
+ value: ${{ steps.compute-projects.outputs.projects }}
+
+runs:
+ using: "composite"
+ steps:
+ - id: compute-projects
+ run: .github/workflows/compute-projects-to-test/compute-projects-to-test.sh ${{ inputs.projects }}
+ shell: bash
diff --git a/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh
new file mode 100755
index 00000000000000..807142668f618b
--- /dev/null
+++ b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh
@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+#
+# This file generates a Buildkite pipeline that triggers the various CI jobs for
+# the LLVM project during pre-commit CI.
+#
+# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
+#
+# As this outputs a yaml file, it's possible to log messages to stderr or
+# prefix with "#".
+
+
+set -eu
+set -o pipefail
+
+# Environment variables script works with:
+
+# Set by GitHub
+: ${GITHUB_OUTPUT:=}
+: ${RUNNER_OS:=}
+
+# Allow users to specify which projects to build.
+all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
+if [ "$#" -ne 0 ]; then
+ wanted_projects="${@}"
+else
+ wanted_projects="${all_projects}"
+fi
+
+# List of files affected by this commit
+: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1...HEAD)}
+
+echo "Files modified:" >&2
+echo "$MODIFIED_FILES" >&2
+modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
+echo "Directories modified:" >&2
+echo "$modified_dirs" >&2
+echo "wanted_projects: $wanted_projects"
+
+function remove-unwanted-projects() {
+ projects=${@}
+ for project in ${projects}; do
+ if echo "$wanted_projects" | tr ' ' '\n' | grep -q -E "^${project}$"; then
+ echo "${project}"
+ fi
+ done
+}
+
+function compute-projects-to-test() {
+ projects=${@}
+ for project in ${projects}; do
+ echo "${project}"
+ case ${project} in
+ lld)
+ for p in bolt cross-project-tests; do
+ echo $p
+ done
+ ;;
+ llvm)
+ for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do
+ echo $p
+ done
+ ;;
+ clang)
+ for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do
+ echo $p
+ done
+ ;;
+ clang-tools-extra)
+ echo libc
+ ;;
+ mlir)
+ echo flang
+ ;;
+ *)
+ # Nothing to do
+ ;;
+ esac
+ done
+}
+
+function add-dependencies() {
+ projects=${@}
+ for project in ${projects}; do
+ echo "${project}"
+ case ${project} in
+ bolt)
+ for p in lld llvm; do
+ echo $p
+ done
+ ;;
+ cross-project-tests)
+ for p in lld clang; do
+ echo $p
+ done
+ ;;
+ clang-tools-extra)
+ for p in llvm clang; do
+ echo $p
+ done
+ ;;
+ compiler-rt|libc|openmp)
+ echo clang lld
+ ;;
+ flang|lldb)
+ for p in llvm clang; do
+ echo $p
+ done
+ ;;
+ lld|mlir|polly)
+ echo llvm
+ ;;
+ *)
+ # Nothing to do
+ ;;
+ esac
+ done
+}
+
+function exclude-linux() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ cross-project-tests) ;; # tests failing
+ lldb) ;; # tests failing
+ openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410
+ *)
+ echo "${project}"
+ ;;
+ esac
+ done
+}
+
+function exclude-windows() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ cross-project-tests) ;; # tests failing
+ compiler-rt) ;; # tests taking too long
+ openmp) ;; # TODO: having trouble with the Perl installation
+ libc) ;; # no Windows support
+ lldb) ;; # tests failing
+ bolt) ;; # tests are not supported yet
+ *)
+ echo "${project}"
+ ;;
+ esac
+ done
+}
+
+# Prints only projects that are both present in $modified_dirs and the passed
+# list.
+function keep-modified-projects() {
+ projects=${@}
+ for project in ${projects}; do
+ if echo "$modified_dirs" | grep -q -E "^${project}$"; then
+ echo "${project}"
+ fi
+ done
+}
+
+function check-targets() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ clang-tools-extra)
+ echo "check-clang-tools"
+ ;;
+ compiler-rt)
+ echo "check-all"
+ ;;
+ cross-project-tests)
+ echo "check-cross-project"
+ ;;
+ lldb)
+ echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
+ ;;
+ pstl)
+ echo "check-all"
+ ;;
+ libclc)
+ echo "check-all"
+ ;;
+ *)
+ echo "check-${project}"
+ ;;
+ esac
+ done
+}
+
+# Generic pipeline for projects that have not defined custom steps.
+#
+# Individual projects should instead define the pre-commit CI tests that suits their
+# needs while letting them run on the infrastructure provided by LLVM.
+
+# Figure out which projects need to be built on each platform
+modified_projects="$(keep-modified-projects ${all_projects})"
+echo "modified_projects: $modified_projects"
+
+if [ "${RUNNER_OS}" = "Linux" ]; then
+ projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
+elif [ "${RUNNER_OS}" = "Windows" ]; then
+ projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
+else
+ echo "Unknown runner OS: $RUNNER_OS"
+fi
+check_targets=$(check-targets $(remove-unwanted-projects ${projects_to_test}) | sort | uniq)
+projects=$(remove-unwanted-projects $(add-dependencies ${projects_to_test}) | sort | uniq)
+
+echo "check-targets=$(echo ${check_targets} | tr ' ' ' ')" >> $GITHUB_OUTPUT
+echo "projects=$(echo ${projects} | tr ' ' ';')" >> $GITHUB_OUTPUT
+
+cat $GITHUB_OUTPUT
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
new file mode 100644
index 00000000000000..4864ab72241dde
--- /dev/null
+++ b/.github/workflows/precommit-windows.yml
@@ -0,0 +1,61 @@
+name: "Windows Precommit Tests"
+
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ branches:
+ - main
+
+concurrency:
+ # Skip intermediate builds: always.
+ # Cancel intermediate builds: only if it is a pull request build.
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+ build-llvm-windows:
+ name: "Build and test LLVM (Windows)"
+ runs-on: windows-2022
+ steps:
+ - name: Setup Windows
+ uses: llvm/actions/setup-windows at main
+ with:
+ arch: amd64
+ - name: Fetch LLVM sources
+ uses: actions/checkout at v4
+ with:
+ fetch-depth: 2
+ - name: Setup ccache
+ uses: hendrikmuhs/ccache-action at v1
+ with:
+ max-size: 500M
+ variant: sccache
+ key: precommit-windows
+ - name: Compute projects to test
+ id: compute-projects
+ uses: ./.github/workflows/compute-projects-to-test
+ with:
+ projects: clang llvm lld
+
+ - name: Configure LLVM
+ shell: bash
+ if: ${{ steps.compute-projects.outputs.check-targets }}
+ run: |
+ cmake -B build -GNinja \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_ENABLE_PROJECTS="${{ steps.compute-projects.outputs.projects }}" \
+ -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+ -DLLVM_ENABLE_ASSERTIONS=ON \
+ -DLLVM_LIT_ARGS="-v --no-progress-bar" \
+ -S llvm
+ - name: Build LLVM
+ if: ${{ steps.compute-projects.outputs.check-targets }}
+ run: |
+ ninja -C build test-depends
+ - name: Check LLVM
+ if: ${{ steps.compute-projects.outputs.check-targets }}
+ run: |
+ ninja -C build "${{ steps.compute-projects.outputs.check-targets }}"
>From c0647188895a3aacf017007a12f313d02b8138b8 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 05:04:31 +0000
Subject: [PATCH 02/11] Test mlir
---
.github/workflows/precommit-windows.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index 4864ab72241dde..e6af5eebf1b1c9 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -37,7 +37,7 @@ jobs:
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
with:
- projects: clang llvm lld
+ projects: clang llvm lld mlir
- name: Configure LLVM
shell: bash
>From d0198b30bad7436d5906534f18e41dc3f8e73b95 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 05:04:44 +0000
Subject: [PATCH 03/11] edit llvm
---
llvm/a | 0
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 llvm/a
diff --git a/llvm/a b/llvm/a
new file mode 100644
index 00000000000000..e69de29bb2d1d6
>From ef76f9c71826b33b0192976d1634844a306fb7bf Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 15:10:50 +0000
Subject: [PATCH 04/11] Fix check-targets
---
.github/workflows/precommit-windows.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index e6af5eebf1b1c9..e4dab2622dcacf 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -58,4 +58,4 @@ jobs:
- name: Check LLVM
if: ${{ steps.compute-projects.outputs.check-targets }}
run: |
- ninja -C build "${{ steps.compute-projects.outputs.check-targets }}"
+ ninja -C build ${{ steps.compute-projects.outputs.check-targets }}
>From 863441090421d5a0b375919821c40b7446affe6c Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 15:11:56 +0000
Subject: [PATCH 05/11] Enable all sub-projects
---
.github/workflows/precommit-windows.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index e4dab2622dcacf..e65a9c7b211ee7 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -36,8 +36,6 @@ jobs:
- name: Compute projects to test
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
- with:
- projects: clang llvm lld mlir
- name: Configure LLVM
shell: bash
>From 60eb5ba4a43ea6c1ef7432d0b82e5d401eb68c5a Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 15:12:27 +0000
Subject: [PATCH 06/11] Disable ccache
---
.github/workflows/precommit-windows.yml | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index e65a9c7b211ee7..476e259b822327 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -27,12 +27,12 @@ jobs:
uses: actions/checkout at v4
with:
fetch-depth: 2
- - name: Setup ccache
- uses: hendrikmuhs/ccache-action at v1
- with:
- max-size: 500M
- variant: sccache
- key: precommit-windows
+ # - name: Setup ccache
+ # uses: hendrikmuhs/ccache-action at v1
+ # with:
+ # max-size: 500M
+ # variant: sccache
+ # key: precommit-windows
- name: Compute projects to test
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
@@ -44,8 +44,6 @@ jobs:
cmake -B build -GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_PROJECTS="${{ steps.compute-projects.outputs.projects }}" \
- -DCMAKE_C_COMPILER_LAUNCHER=sccache \
- -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_LIT_ARGS="-v --no-progress-bar" \
-S llvm
>From 896c30892db3e007de4742700640209c8d08cf8c Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 23:54:29 +0000
Subject: [PATCH 07/11] Revert "Enable all sub-projects"
This reverts commit 863441090421d5a0b375919821c40b7446affe6c.
---
.github/workflows/precommit-windows.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index 476e259b822327..fe5953f39552a3 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -36,6 +36,8 @@ jobs:
- name: Compute projects to test
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
+ with:
+ projects: clang llvm lld mlir
- name: Configure LLVM
shell: bash
>From 12ec618836c9989cb55302d9e4b36859ecb9908e Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 23:55:25 +0000
Subject: [PATCH 08/11] Enable all projects but flang
---
.github/workflows/precommit-windows.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index fe5953f39552a3..024ead8805bc7e 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -37,7 +37,7 @@ jobs:
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
with:
- projects: clang llvm lld mlir
+ projects: clang llvm lld mlir clang-tools-extra polly
- name: Configure LLVM
shell: bash
>From 7b4014b72f85c4b908ebfd27f730d4e3a19874d6 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 23:56:11 +0000
Subject: [PATCH 09/11] Remove polly
---
.github/workflows/precommit-windows.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index 024ead8805bc7e..89b1424dfd8e9c 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -37,7 +37,7 @@ jobs:
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
with:
- projects: clang llvm lld mlir clang-tools-extra polly
+ projects: clang llvm lld mlir clang-tools-extra
- name: Configure LLVM
shell: bash
>From 87e2307ba3311450e5bd28472115f9bc4f810f28 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 23:56:55 +0000
Subject: [PATCH 10/11] Remove clang-tools-extra
---
.github/workflows/precommit-windows.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index 89b1424dfd8e9c..fe5953f39552a3 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -37,7 +37,7 @@ jobs:
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
with:
- projects: clang llvm lld mlir clang-tools-extra
+ projects: clang llvm lld mlir
- name: Configure LLVM
shell: bash
>From 378662b6376cafad38d4a5e18c484cca1fe91ae9 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 17 Feb 2024 23:57:36 +0000
Subject: [PATCH 11/11] Remove mlir
---
.github/workflows/precommit-windows.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml
index fe5953f39552a3..09ffe025b62de3 100644
--- a/.github/workflows/precommit-windows.yml
+++ b/.github/workflows/precommit-windows.yml
@@ -37,7 +37,7 @@ jobs:
id: compute-projects
uses: ./.github/workflows/compute-projects-to-test
with:
- projects: clang llvm lld mlir
+ projects: clang llvm lld
- name: Configure LLVM
shell: bash
More information about the llvm-commits
mailing list