[llvm-branch-commits] [CI] Move CI over to new project computation script (PR #132642)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Mar 23 15:44:23 PDT 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-github-workflow
Author: Aiden Grossman (boomanaiden154)
<details>
<summary>Changes</summary>
This patch migrates the CI over to the new compute_projects.py script
for calculating what projects need to be tested based on a change to
LLVM.
---
Full diff: https://github.com/llvm/llvm-project/pull/132642.diff
3 Files Affected:
- (removed) .ci/compute-projects.sh (-194)
- (modified) .ci/generate-buildkite-pipeline-premerge (+8-14)
- (modified) .github/workflows/premerge.yaml (+19-71)
``````````diff
diff --git a/.ci/compute-projects.sh b/.ci/compute-projects.sh
deleted file mode 100644
index 32baf26b4f0a0..0000000000000
--- a/.ci/compute-projects.sh
+++ /dev/null
@@ -1,194 +0,0 @@
-#!/usr/bin/env bash
-#===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-
-#
-# This file contains functions to compute which projects should be built by CI
-# systems and is intended to provide common functionality applicable across
-# multiple systems during a transition period.
-#
-
-function compute-projects-to-test() {
- isForWindows=$1
- shift
- projects=${@}
- for project in ${projects}; do
- echo "${project}"
- case ${project} in
- lld)
- for p in bolt cross-project-tests; do
- echo $p
- done
- ;;
- llvm)
- for p in bolt clang clang-tools-extra lld lldb mlir polly; do
- echo $p
- done
- # Flang is not stable in Windows CI at the moment
- if [[ $isForWindows == 0 ]]; then
- echo flang
- fi
- ;;
- clang)
- # lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
- for p in clang-tools-extra compiler-rt cross-project-tests; do
- echo $p
- done
- ;;
- clang-tools-extra)
- echo libc
- ;;
- mlir)
- # Flang is not stable in Windows CI at the moment
- if [[ $isForWindows == 0 ]]; then
- echo flang
- fi
- ;;
- *)
- # Nothing to do
- ;;
- esac
- done
-}
-
-function compute-runtimes-to-test() {
- projects=${@}
- for project in ${projects}; do
- case ${project} in
- clang)
- for p in libcxx libcxxabi libunwind; do
- echo $p
- done
- ;;
- *)
- # Nothing to do
- ;;
- esac
- done
-}
-
-function add-dependencies() {
- projects=${@}
- for project in ${projects}; do
- echo "${project}"
- case ${project} in
- bolt)
- for p in clang lld llvm; do
- echo $p
- done
- ;;
- cross-project-tests)
- for p in lld clang; do
- echo $p
- done
- ;;
- clang-tools-extra)
- for p in llvm clang; do
- echo $p
- done
- ;;
- compiler-rt|libc|openmp)
- echo clang lld
- ;;
- flang|lldb|libclc)
- for p in llvm clang; do
- echo $p
- done
- ;;
- lld|mlir|polly)
- echo llvm
- ;;
- *)
- # Nothing to do
- ;;
- esac
- done
-}
-
-function exclude-linux() {
- projects=${@}
- for project in ${projects}; do
- case ${project} in
- cross-project-tests) ;; # tests failing
- openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410
- *)
- echo "${project}"
- ;;
- esac
- done
-}
-
-function exclude-windows() {
- projects=${@}
- for project in ${projects}; do
- case ${project} in
- cross-project-tests) ;; # tests failing
- compiler-rt) ;; # tests taking too long
- openmp) ;; # TODO: having trouble with the Perl installation
- libc) ;; # no Windows support
- lldb) ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
- bolt) ;; # tests are not supported yet
- *)
- echo "${project}"
- ;;
- esac
- done
-}
-
-# Prints only projects that are both present in $modified_dirs and the passed
-# list.
-function keep-modified-projects() {
- projects=${@}
- for project in ${projects}; do
- if echo "$modified_dirs" | grep -q -E "^${project}$"; then
- echo "${project}"
- fi
- done
-}
-
-function check-targets() {
- # Do not use "check-all" here because if there is "check-all" plus a
- # project specific target like "check-clang", that project's tests
- # will be run twice.
- projects=${@}
- for project in ${projects}; do
- case ${project} in
- clang-tools-extra)
- echo "check-clang-tools"
- ;;
- compiler-rt)
- echo "check-compiler-rt"
- ;;
- cross-project-tests)
- echo "check-cross-project"
- ;;
- libcxx)
- echo "check-cxx"
- ;;
- libcxxabi)
- echo "check-cxxabi"
- ;;
- libunwind)
- echo "check-unwind"
- ;;
- lldb)
- echo "check-lldb"
- ;;
- pstl)
- # Currently we do not run pstl tests in CI.
- ;;
- libclc)
- # Currently there is no testing for libclc.
- ;;
- *)
- echo "check-${project}"
- ;;
- esac
- done
-}
-
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index e547afaeb722f..296b3b605f49b 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -52,8 +52,6 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
echo "Directories modified:" >&2
echo "$modified_dirs" >&2
-. ./.ci/compute-projects.sh
-
# Project specific pipelines.
# If libc++ or one of the runtimes directories changed.
@@ -73,20 +71,16 @@ fi
# needs while letting them run on the infrastructure provided by LLVM.
# Figure out which projects need to be built on each platform
-all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-modified_projects="$(keep-modified-projects ${all_projects})"
-
-linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
-linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
-linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
+source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py Linux)
+linux_projects=${projects_to_build}
+linux_check_targets=${project_check_targets}
+linux_runtimes=${runtimes_to_build}
+linux_runtime_check_targets=${runtimes_check_targets}
-linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
-linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
-linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
+source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py Windows)
+windows_projects=${projects_to_build}
+windows_check_targets=${project_check_targets}
-windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
-windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
-windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
# Generate the appropriate pipeline
if [[ "${linux_projects}" != "" ]]; then
diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml
index c488421d37450..deb4cf7d70447 100644
--- a/.github/workflows/premerge.yaml
+++ b/.github/workflows/premerge.yaml
@@ -49,39 +49,22 @@ jobs:
run: |
git config --global --add safe.directory '*'
- modified_files=$(git diff --name-only HEAD~1...HEAD)
- modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
+ source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py)
- echo $modified_files
- echo $modified_dirs
-
- . ./.ci/compute-projects.sh
-
- all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
- modified_projects="$(keep-modified-projects ${all_projects})"
-
- linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
- linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
- linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
-
- linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
- linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
- linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
-
- if [[ "${linux_projects}" == "" ]]; then
+ if [[ "${projects_to_build}" == "" ]]; then
echo "No projects to build"
exit 0
fi
- echo "Building projects: ${linux_projects}"
- echo "Running project checks targets: ${linux_check_targets}"
- echo "Building runtimes: ${linux_runtimes}"
- echo "Running runtimes checks targets: ${linux_runtime_check_targets}"
+ echo "Building projects: ${projects_to_build}"
+ echo "Running project checks targets: ${project_check_targets}"
+ echo "Building runtimes: ${runtimes_to_build}"
+ echo "Running runtimes checks targets: ${runtimes_check_targets}"
export CC=/opt/llvm/bin/clang
export CXX=/opt/llvm/bin/clang++
- ./.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"
+ ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}"
premerge-checks-windows:
name: Windows Premerge Checks (Test Only - Please Ignore Results)
@@ -105,30 +88,17 @@ jobs:
- name: Compute Projects
id: vars
run: |
- modified_files=$(git diff --name-only HEAD~1...HEAD)
- modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort | uniq)
-
- echo $modified_files
- echo $modified_dirs
-
- . ./.ci/compute-projects.sh
+ source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py)
- all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
- modified_projects="$(keep-modified-projects ${all_projects})"
-
- windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
- windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ' ')
- windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ';')
-
- if [[ "${windows_projects}" == "" ]]; then
+ if [[ "${projects_to_build}" == "" ]]; then
echo "No projects to build"
fi
- echo "Building projects: ${windows_projects}"
- echo "Running project checks targets: ${windows_check_targets}"
+ echo "Building projects: ${projects_to_build}"
+ echo "Running project checks targets: ${project_check_targets}"
- echo "windows-projects=${windows_projects}" >> $GITHUB_OUTPUT
- echo "windows-check-targets=${windows_check_targets}" >> $GITHUB_OUTPUT
+ echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT
+ echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT
- name: Build and Test
# Mark the job as a success even if the step fails so that people do
# not get notified while the new premerge pipeline is in an
@@ -165,37 +135,15 @@ jobs:
uses: llvm/actions/install-ninja at main
- name: Build and Test
run: |
- modified_files=$(git diff --name-only HEAD~1...HEAD)
- modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
-
- echo $modified_files
- echo $modified_dirs
-
- . ./.ci/compute-projects.sh
-
- all_projects="clang clang-tools-extra lld lldb llvm mlir"
- modified_projects="$(keep-modified-projects ${all_projects})"
-
- # We have to disable the runtimes builds due to https://github.com/llvm/llvm-project/issues/90568
- # and the lldb tests depend on libcxx, so we need to skip them.
- mac_check_targets=$(check-targets ${modified_projects} | sort | uniq | tr '\n' ' ' | sed -e 's/check-lldb //g')
- mac_projects=$(add-dependencies ${modified_projects} | sort | uniq | tr '\n' ' ')
-
- mac_runtimes_to_test=$(compute-runtimes-to-test ${modified_projects})
- mac_runtime_check_targets=$(check-targets ${mac_runtimes_to_test} | sort | uniq | tr '\n' ' ')
- mac_runtimes=$(echo ${mac_runtimes_to_test} | tr ' ' '\n' | sort | uniq | tr '\n' ' ')
+ source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py)
- if [[ "${mac_projects}" == "" ]]; then
+ if [[ "${projects_to_build}" == "" ]]; then
echo "No projects to build"
exit 0
fi
- echo "Projects to test: ${modified_projects}"
- echo "Runtimes to test: ${mac_runtimes_to_test}"
- echo "Building projects: ${mac_projects}"
- echo "Running project checks targets: ${mac_check_targets}"
- echo "Building runtimes: ${mac_runtimes}"
- echo "Running runtimes checks targets: ${mac_runtime_check_targets}"
+ echo "Building projects: ${projects_to_build}"
+ echo "Running project checks targets: ${project_check_targets}"
# -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for
# https://github.com/llvm/llvm-project/issues/81967
@@ -203,7 +151,7 @@ jobs:
cmake -G Ninja \
-B build \
-S llvm \
- -DLLVM_ENABLE_PROJECTS="$(echo ${mac_projects} | tr ' ' ';')" \
+ -DLLVM_ENABLE_PROJECTS="${projects_to_build}" \
-DLLVM_DISABLE_ASSEMBLY_FILES=ON \
-DCMAKE_BUILD_TYPE=Release \
-DLLDB_INCLUDE_TESTS=OFF \
@@ -212,4 +160,4 @@ jobs:
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
# The libcxx tests fail, so we are skipping the runtime targets.
- ninja -C build $mac_check_targets
+ ninja -C build ${project_check_targets}
``````````
</details>
https://github.com/llvm/llvm-project/pull/132642
More information about the llvm-branch-commits
mailing list