[llvm] Scheduled bildkite pipeline generation (PR #65574)
Mikhail Goncharov via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 6 23:44:48 PDT 2023
https://github.com/metaflow created https://github.com/llvm/llvm-project/pull/65574:
- fixed build for linux (clang was missing)
- removed /monolithic-.. from build directory - it does not add anything and makes path longer for windows which is not great;
- added env-based configuration to control cache and agent targeting;
- print (s)ccache stats to file not to pullute normal log.
>From 5e35b73fdc73b264ccd963a869c6251094edbf7e Mon Sep 17 00:00:00 2001
From: Mikhail Goncharov <goncharov.mikhail at gmail.com>
Date: Wed, 6 Sep 2023 18:44:09 +0200
Subject: [PATCH] Scheduled bildkite pipeline generation
- fixed build for linux (clang was missing)
- removed /monolithic-.. from build directory - it does not add anything
and makes path longer for windows which is not great;
- added env-based configuration to control cache and agent targeting;
- print (s)ccache stats to file not to pullute normal log.
---
.ci/generate-buildkite-pipeline-premerge | 4 +-
.ci/generate-buildkite-pipeline-scheduled | 46 ++++++++++++++---------
.ci/monolithic-linux.sh | 13 +++++--
.ci/monolithic-windows.sh | 12 ++++--
4 files changed, 49 insertions(+), 26 deletions(-)
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index c0071dcc7f312b8..e889f3d725bf709 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -240,7 +240,7 @@ if [[ "${linux_projects}" != "" ]]; then
artifact_paths:
- artifacts/**/*
- '*_result.json'
- - 'build/monolithic-linux/test-results.xml'
+ - 'build/test-results.xml'
agents: ${LINUX_AGENTS}
retry:
automatic:
@@ -263,7 +263,7 @@ if [[ "${windows_projects}" != "" ]]; then
artifact_paths:
- artifacts/**/*
- '*_result.json'
- - 'build/monolithic-windows/test-results.xml'
+ - 'build/test-results.xml'
agents: ${WINDOWS_AGENTS}
retry:
automatic:
diff --git a/.ci/generate-buildkite-pipeline-scheduled b/.ci/generate-buildkite-pipeline-scheduled
index 06607ce617941f3..d5a6bce6458c462 100755
--- a/.ci/generate-buildkite-pipeline-scheduled
+++ b/.ci/generate-buildkite-pipeline-scheduled
@@ -14,6 +14,18 @@
# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
#
+set -eu
+set -o pipefail
+
+# Filter rules for generic windows tests
+: ${WINDOWS_AGENTS:='{"queue": "windows"}'}
+# Filter rules for generic linux tests
+: ${LINUX_AGENTS:='{"queue": "linux"}'}
+# Set by buildkite
+: ${BUILDKITE_MESSAGE:=}
+: ${BUILDKITE_COMMIT:=}
+: ${BUILDKITE_BRANCH:=}
+
cat <<EOF
steps:
- trigger: "libcxx-ci"
@@ -28,42 +40,42 @@ steps:
commit: "${BUILDKITE_COMMIT}"
branch: "${BUILDKITE_BRANCH}"
- - label: ':linux: x64 Debian'
+ - label: ':linux: Linux x64'
artifact_paths:
- - '*_result.json'
- - 'build/monolithic-linux/test-results.xml'
- agents:
- queue: 'linux'
+ - artifacts/**/*
+ - '*_result.json'
+ - 'build/test-results.xml'
+ agents: ${LINUX_AGENTS}
retry:
automatic:
- exit_status: -1 # Agent was lost
limit: 2
- - exit_status: 255
- limit: 2 # Forced agent shutdown
+ - exit_status: 255 # Forced agent shutdown
+ limit: 2
timeout_in_minutes: 120
env:
CC: 'clang'
CXX: 'clang++'
commands:
- - './.ci/monolithic-linux.sh "bolt;clang-tools-extra;compiler-rt;flang;libc;libclc;lld;llvm;mlir;polly;pstl" "check-all"'
+ - ./.ci/monolithic-linux.sh "bolt;clang;clang-tools-extra;compiler-rt;flang;libc;libclc;lld;llvm;mlir;polly;pstl" "check-all"
- - label: ':windows: x64 Windows'
+ - label: ':windows: Windows x64'
artifact_paths:
- - '*_result.json'
- - 'build/monolithic-windows/test-results.xml'
- agents:
- queue: 'windows'
+ - artifacts/**/*
+ - '*_result.json'
+ - 'build/test-results.xml'
+ agents: ${WINDOWS_AGENTS}
retry:
automatic:
- exit_status: -1 # Agent was lost
limit: 2
- - exit_status: 255
- limit: 2 # Forced agent shutdown
+ - exit_status: 255 # Forced agent shutdown
+ limit: 2
timeout_in_minutes: 150
env:
CC: 'cl'
CXX: 'cl'
LD: 'link'
commands:
- - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
- - 'bash .ci/monolithic-windows.sh "clang-tools-extra;flang;libclc;lld;llvm;mlir;polly;pstl" "check-all"'
+ - C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64
+ - bash .ci/monolithic-windows.sh "clang-tools-extra;flang;libclc;lld;llvm;mlir;polly;pstl;clang" "check-all"
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 9054d70e3b01380..03cf38b90c08fd8 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -17,14 +17,19 @@ set -ex
set -o pipefail
MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
-BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-linux}"
-
+BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
rm -rf ${BUILD_DIR}
ccache --zero-stats
-ccache --show-config
+
+if [[ -n "${CLEAR_CACHE:-}" ]]; then
+ echo "clearing cache"
+ ccache --clear
+fi
+
function show-stats {
- ccache --print-stats
+ mkdir -p artifacts
+ ccache --print-stats > artifacts/ccache_stats.txt
}
trap show-stats EXIT
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index da15f7c1923905b..83be9d04b992781 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -17,13 +17,19 @@ set -ex
set -o pipefail
MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
-BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-windows}"
+BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
rm -rf ${BUILD_DIR}
+if [[ -n "${CLEAR_CACHE:-}" ]]; then
+ echo "clearing sccache"
+ rm -rf "$SCCACHE_DIR"
+fi
+
sccache --zero-stats
function show-stats {
- sccache --show-stats
+ mkdir -p artifacts
+ sccache --show-stats >> artifacts/sccache_stats.txt
}
trap show-stats EXIT
@@ -45,4 +51,4 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
-D CMAKE_CXX_COMPILER_LAUNCHER=sccache
echo "--- ninja"
-ninja -C ${BUILD_DIR} ${targets}
+ninja -C "${BUILD_DIR}" "${targets}"
More information about the llvm-commits
mailing list