[llvm] Scheduled buildkite pipeline generation (PR #65574)

[Mikhail Goncharov via llvm-commits]

https://github.com/metaflow updated https://github.com/llvm/llvm-project/pull/65574:

>From 428fae33e94a12d40fcbc961e460c930f8f9ebab Mon Sep 17 00:00:00 2001
From: Mikhail Goncharov <goncharov.mikhail at gmail.com>
Date: Mon, 11 Sep 2023 11:59:39 +0200
Subject: [PATCH] update buildkite pipeline generation

- fixed build for linux (clang was missing)

- removed /monolithic-.. from build directory - it does not add anything
  and makes path longer for windows which is not great;

- added env-based configuration to control cache and agent targeting;

- print (s)ccache stats to file not to pullute normal log.
---
 .ci/generate-buildkite-pipeline-premerge  |  8 ++--
 .ci/generate-buildkite-pipeline-scheduled | 46 ++++++++++++++---------
 .ci/monolithic-linux.sh                   | 13 +++++--
 .ci/monolithic-windows.sh                 | 12 ++++--
 4 files changed, 51 insertions(+), 28 deletions(-)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index c0071dcc7f312b8..7d14aed355d42d6 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -239,8 +239,8 @@ if [[ "${linux_projects}" != "" ]]; then
 - label: ':linux: Linux x64'
   artifact_paths:
   - artifacts/**/*
-  - '*_result.json'
-  - 'build/monolithic-linux/test-results.xml'
+  - *_result.json
+  - build/test-results.xml
   agents: ${LINUX_AGENTS}
   retry:
     automatic:
@@ -262,8 +262,8 @@ if [[ "${windows_projects}" != "" ]]; then
 - label: ':windows: Windows x64'
   artifact_paths:
   - artifacts/**/*
-  - '*_result.json'
-  - 'build/monolithic-windows/test-results.xml'
+  - *_result.json
+  - build/test-results.xml
   agents: ${WINDOWS_AGENTS}
   retry:
     automatic:
diff --git a/.ci/generate-buildkite-pipeline-scheduled b/.ci/generate-buildkite-pipeline-scheduled
index 06607ce617941f3..6ad26aeb6e385df 100755
--- a/.ci/generate-buildkite-pipeline-scheduled
+++ b/.ci/generate-buildkite-pipeline-scheduled
@@ -14,6 +14,18 @@
 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
 #
 
+set -eu
+set -o pipefail
+
+# Filter rules for generic windows tests
+: ${WINDOWS_AGENTS:='{"queue": "windows"}'}
+# Filter rules for generic linux tests
+: ${LINUX_AGENTS:='{"queue": "linux"}'}
+# Set by buildkite
+: ${BUILDKITE_MESSAGE:=}
+: ${BUILDKITE_COMMIT:=}
+: ${BUILDKITE_BRANCH:=}
+
 cat <<EOF
 steps:
   - trigger: "libcxx-ci"
@@ -28,42 +40,42 @@ steps:
       commit: "${BUILDKITE_COMMIT}"
       branch: "${BUILDKITE_BRANCH}"
 
-  - label: ':linux: x64 Debian'
+  - label: ':linux: Linux x64'
     artifact_paths:
-      - '*_result.json'
-      - 'build/monolithic-linux/test-results.xml'
-    agents:
-      queue: 'linux'
+    - artifacts/**/*
+    - '*_result.json'
+    - 'build/test-results.xml'
+    agents: ${LINUX_AGENTS}
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
-        - exit_status: 255
-          limit: 2 # Forced agent shutdown
+        - exit_status: 255 # Forced agent shutdown
+          limit: 2
     timeout_in_minutes: 120
     env:
       CC: 'clang'
       CXX: 'clang++'
     commands:
-      - './.ci/monolithic-linux.sh "bolt;clang-tools-extra;compiler-rt;flang;libc;libclc;lld;llvm;mlir;polly;pstl" "check-all"'
+    - ./.ci/monolithic-linux.sh "bolt;clang;clang-tools-extra;compiler-rt;flang;libc;libclc;lld;llvm;mlir;polly;pstl" "check-all"
 
-  - label: ':windows: x64 Windows'
+  - label: ':windows: Windows x64'
     artifact_paths:
-      - '*_result.json'
-      - 'build/monolithic-windows/test-results.xml'
-    agents:
-      queue: 'windows'
+    - artifacts/**/*
+    - '*_result.json'
+    - 'build/test-results.xml'
+    agents: ${WINDOWS_AGENTS}
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
-        - exit_status: 255
-          limit: 2 # Forced agent shutdown
+        - exit_status: 255 # Forced agent shutdown
+          limit: 2
     timeout_in_minutes: 150
     env:
       CC: 'cl'
       CXX: 'cl'
       LD: 'link'
     commands:
-      - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
-      - 'bash .ci/monolithic-windows.sh "clang-tools-extra;flang;libclc;lld;llvm;mlir;polly;pstl" "check-all"'
+    - C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64
+    - bash .ci/monolithic-windows.sh "clang;clang-tools-extra;flang;libclc;lld;llvm;mlir;polly;pstl" "check-all"
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 9054d70e3b01380..03cf38b90c08fd8 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -17,14 +17,19 @@ set -ex
 set -o pipefail
 
 MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
-BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-linux}"
-
+BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
 rm -rf ${BUILD_DIR}
 
 ccache --zero-stats
-ccache --show-config
+
+if [[ -n "${CLEAR_CACHE:-}" ]]; then
+  echo "clearing cache"
+  ccache --clear
+fi
+
 function show-stats {
-  ccache --print-stats
+  mkdir -p artifacts
+  ccache --print-stats > artifacts/ccache_stats.txt
 }
 trap show-stats EXIT
 
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index da15f7c1923905b..83be9d04b992781 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -17,13 +17,19 @@ set -ex
 set -o pipefail
 
 MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
-BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-windows}"
+BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
 
 rm -rf ${BUILD_DIR}
 
+if [[ -n "${CLEAR_CACHE:-}" ]]; then
+  echo "clearing sccache"
+  rm -rf "$SCCACHE_DIR"
+fi
+
 sccache --zero-stats
 function show-stats {
-  sccache --show-stats
+  mkdir -p artifacts
+  sccache --show-stats >> artifacts/sccache_stats.txt
 }
 trap show-stats EXIT
 
@@ -45,4 +51,4 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D CMAKE_CXX_COMPILER_LAUNCHER=sccache
 
 echo "--- ninja"
-ninja -C ${BUILD_DIR} ${targets}
+ninja -C "${BUILD_DIR}" "${targets}"