[llvm] [CI] Refactor common functionality into separate script (PR #119530)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 12 22:48:47 PST 2024
https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/119530
>From 6ffa419968dd2c10c11f66c380fba2f77a18767a Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Wed, 11 Dec 2024 09:38:33 +0000
Subject: [PATCH 1/4] [CI] Refactor common functionality into separate script
This patch refactors some common functionality present in the CI scripts
to a separate shell script. This is mainly intended to make it easier to
reuse this functionality inside of a Github Actions pipeline as we make
the switch.
---
.ci/compute-projects.sh | 165 +++++++++++++++++++++
.ci/generate-buildkite-pipeline-premerge | 179 +----------------------
2 files changed, 166 insertions(+), 178 deletions(-)
create mode 100644 .ci/compute-projects.sh
diff --git a/.ci/compute-projects.sh b/.ci/compute-projects.sh
new file mode 100644
index 00000000000000..b33158339ef3fd
--- /dev/null
+++ b/.ci/compute-projects.sh
@@ -0,0 +1,165 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+#
+# This file contains functions to compute which projects should be built by CI
+# systems and is intended to provide common functionality applicable across
+# multiple systems during a transition period.
+#
+
+function compute-projects-to-test() {
+ isForWindows=$1
+ shift
+ projects=${@}
+ for project in ${projects}; do
+ echo "${project}"
+ case ${project} in
+ lld)
+ for p in bolt cross-project-tests; do
+ echo $p
+ done
+ ;;
+ llvm)
+ for p in bolt clang clang-tools-extra lld lldb mlir polly; do
+ echo $p
+ done
+ # Flang is not stable in Windows CI at the moment
+ if [[ $isForWindows == 0 ]]; then
+ echo flang
+ fi
+ ;;
+ clang)
+ # lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
+ for p in clang-tools-extra compiler-rt cross-project-tests; do
+ echo $p
+ done
+ ;;
+ clang-tools-extra)
+ echo libc
+ ;;
+ mlir)
+ # Flang is not stable in Windows CI at the moment
+ if [[ $isForWindows == 0 ]]; then
+ echo flang
+ fi
+ ;;
+ *)
+ # Nothing to do
+ ;;
+ esac
+ done
+}
+
+function compute-runtimes-to-test() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ clang)
+ for p in libcxx libcxxabi libunwind; do
+ echo $p
+ done
+ ;;
+ *)
+ # Nothing to do
+ ;;
+ esac
+ done
+}
+
+function add-dependencies() {
+ projects=${@}
+ for project in ${projects}; do
+ echo "${project}"
+ case ${project} in
+ bolt)
+ for p in clang lld llvm; do
+ echo $p
+ done
+ ;;
+ cross-project-tests)
+ for p in lld clang; do
+ echo $p
+ done
+ ;;
+ clang-tools-extra)
+ for p in llvm clang; do
+ echo $p
+ done
+ ;;
+ compiler-rt|libc|openmp)
+ echo clang lld
+ ;;
+ flang|lldb|libclc)
+ for p in llvm clang; do
+ echo $p
+ done
+ ;;
+ lld|mlir|polly)
+ echo llvm
+ ;;
+ *)
+ # Nothing to do
+ ;;
+ esac
+ done
+}
+
+# Prints only projects that are both present in $modified_dirs and the passed
+# list.
+function keep-modified-projects() {
+ projects=${@}
+ modified_dirs=${1}
+ for project in ${projects}; do
+ if echo "$modified_dirs" | grep -q -E "^${project}$"; then
+ echo "${project}"
+ fi
+ done
+}
+
+function check-targets() {
+ # Do not use "check-all" here because if there is "check-all" plus a
+ # project specific target like "check-clang", that project's tests
+ # will be run twice.
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ clang-tools-extra)
+ echo "check-clang-tools"
+ ;;
+ compiler-rt)
+ echo "check-compiler-rt"
+ ;;
+ cross-project-tests)
+ echo "check-cross-project"
+ ;;
+ libcxx)
+ echo "check-cxx"
+ ;;
+ libcxxabi)
+ echo "check-cxxabi"
+ ;;
+ libunwind)
+ echo "check-unwind"
+ ;;
+ lldb)
+ echo "check-lldb"
+ ;;
+ pstl)
+ # Currently we do not run pstl tests in CI.
+ ;;
+ libclc)
+ # Currently there is no testing for libclc.
+ ;;
+ *)
+ echo "check-${project}"
+ ;;
+ esac
+ done
+}
+
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 190dd1e5ba5af0..ddcaca813be3c0 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -52,184 +52,7 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
echo "Directories modified:" >&2
echo "$modified_dirs" >&2
-function compute-projects-to-test() {
- isForWindows=$1
- shift
- projects=${@}
- for project in ${projects}; do
- echo "${project}"
- case ${project} in
- lld)
- for p in bolt cross-project-tests; do
- echo $p
- done
- ;;
- llvm)
- for p in bolt clang clang-tools-extra lld lldb mlir polly; do
- echo $p
- done
- # Flang is not stable in Windows CI at the moment
- if [[ $isForWindows == 0 ]]; then
- echo flang
- fi
- ;;
- clang)
- # lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
- for p in clang-tools-extra compiler-rt cross-project-tests; do
- echo $p
- done
- ;;
- clang-tools-extra)
- echo libc
- ;;
- mlir)
- # Flang is not stable in Windows CI at the moment
- if [[ $isForWindows == 0 ]]; then
- echo flang
- fi
- ;;
- *)
- # Nothing to do
- ;;
- esac
- done
-}
-
-function compute-runtimes-to-test() {
- projects=${@}
- for project in ${projects}; do
- case ${project} in
- clang)
- for p in libcxx libcxxabi libunwind; do
- echo $p
- done
- ;;
- *)
- # Nothing to do
- ;;
- esac
- done
-}
-
-function add-dependencies() {
- projects=${@}
- for project in ${projects}; do
- echo "${project}"
- case ${project} in
- bolt)
- for p in clang lld llvm; do
- echo $p
- done
- ;;
- cross-project-tests)
- for p in lld clang; do
- echo $p
- done
- ;;
- clang-tools-extra)
- for p in llvm clang; do
- echo $p
- done
- ;;
- compiler-rt|libc|openmp)
- echo clang lld
- ;;
- flang|lldb|libclc)
- for p in llvm clang; do
- echo $p
- done
- ;;
- lld|mlir|polly)
- echo llvm
- ;;
- *)
- # Nothing to do
- ;;
- esac
- done
-}
-
-function exclude-linux() {
- projects=${@}
- for project in ${projects}; do
- case ${project} in
- cross-project-tests) ;; # tests failing
- openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410
- *)
- echo "${project}"
- ;;
- esac
- done
-}
-
-function exclude-windows() {
- projects=${@}
- for project in ${projects}; do
- case ${project} in
- cross-project-tests) ;; # tests failing
- compiler-rt) ;; # tests taking too long
- openmp) ;; # TODO: having trouble with the Perl installation
- libc) ;; # no Windows support
- lldb) ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
- bolt) ;; # tests are not supported yet
- *)
- echo "${project}"
- ;;
- esac
- done
-}
-
-# Prints only projects that are both present in $modified_dirs and the passed
-# list.
-function keep-modified-projects() {
- projects=${@}
- for project in ${projects}; do
- if echo "$modified_dirs" | grep -q -E "^${project}$"; then
- echo "${project}"
- fi
- done
-}
-
-function check-targets() {
- # Do not use "check-all" here because if there is "check-all" plus a
- # project specific target like "check-clang", that project's tests
- # will be run twice.
- projects=${@}
- for project in ${projects}; do
- case ${project} in
- clang-tools-extra)
- echo "check-clang-tools"
- ;;
- compiler-rt)
- echo "check-compiler-rt"
- ;;
- cross-project-tests)
- echo "check-cross-project"
- ;;
- libcxx)
- echo "check-cxx"
- ;;
- libcxxabi)
- echo "check-cxxabi"
- ;;
- libunwind)
- echo "check-unwind"
- ;;
- lldb)
- echo "check-lldb"
- ;;
- pstl)
- # Currently we do not run pstl tests in CI.
- ;;
- libclc)
- # Currently there is no testing for libclc.
- ;;
- *)
- echo "check-${project}"
- ;;
- esac
- done
-}
+. compute-projects.sh
# Project specific pipelines.
>From d858ae3560e4385a653cf22972ac4c8557b0313e Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Wed, 11 Dec 2024 09:42:31 +0000
Subject: [PATCH 2/4] Maybe fix paths
---
.ci/generate-buildkite-pipeline-premerge | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index ddcaca813be3c0..31186bef9c8933 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -52,7 +52,7 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
echo "Directories modified:" >&2
echo "$modified_dirs" >&2
-. compute-projects.sh
+. ./ci/compute-projects.sh
# Project specific pipelines.
>From a8eae851d0ce8621cc10cfa0b9df87b2d35aaa48 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Wed, 11 Dec 2024 09:43:24 +0000
Subject: [PATCH 3/4] Fix spelling
---
.ci/generate-buildkite-pipeline-premerge | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 31186bef9c8933..9d9ca321839449 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -52,7 +52,7 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
echo "Directories modified:" >&2
echo "$modified_dirs" >&2
-. ./ci/compute-projects.sh
+. ./.ci/compute-projects.sh
# Project specific pipelines.
>From 1c020bba94fe2e70820a9d2e1221dd73b9676332 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Wed, 11 Dec 2024 09:44:40 +0000
Subject: [PATCH 4/4] Add missing functions
---
.ci/compute-projects.sh | 31 ++++++++++++++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
diff --git a/.ci/compute-projects.sh b/.ci/compute-projects.sh
index b33158339ef3fd..32baf26b4f0a05 100644
--- a/.ci/compute-projects.sh
+++ b/.ci/compute-projects.sh
@@ -110,11 +110,40 @@ function add-dependencies() {
done
}
+function exclude-linux() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ cross-project-tests) ;; # tests failing
+ openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410
+ *)
+ echo "${project}"
+ ;;
+ esac
+ done
+}
+
+function exclude-windows() {
+ projects=${@}
+ for project in ${projects}; do
+ case ${project} in
+ cross-project-tests) ;; # tests failing
+ compiler-rt) ;; # tests taking too long
+ openmp) ;; # TODO: having trouble with the Perl installation
+ libc) ;; # no Windows support
+ lldb) ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
+ bolt) ;; # tests are not supported yet
+ *)
+ echo "${project}"
+ ;;
+ esac
+ done
+}
+
# Prints only projects that are both present in $modified_dirs and the passed
# list.
function keep-modified-projects() {
projects=${@}
- modified_dirs=${1}
for project in ${projects}; do
if echo "$modified_dirs" | grep -q -E "^${project}$"; then
echo "${project}"
More information about the llvm-commits
mailing list