[llvm] [ci] limit parallel windows compile jobs to 24 (PR #93329)

[Lucile Rose Nihlen via llvm-commits]

https://github.com/lnihlen updated https://github.com/llvm/llvm-project/pull/93329

>From 5f07f44f2d3d686603fc3293815b99089474fb72 Mon Sep 17 00:00:00 2001
From: Lucile Nihlen <luci.the.rose at gmail.com>
Date: Fri, 24 May 2024 17:55:49 +0000
Subject: [PATCH 1/6] [ci] limit parallel windows compile jobs to 24

---
 .ci/monolithic-windows.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 4fd88ea81c84a..d31a913826d7e 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -44,6 +44,8 @@ pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 # see https://github.com/llvm/llvm-project/pull/82393 and
 # https://discourse.llvm.org/t/rfc-future-of-windows-pre-commit-ci/76840/40
 # for further information.
+# We limit the number of parallel compile jobs to 24 control memory
+# consumption and improve build reliability.
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
       -G Ninja \
@@ -58,7 +60,9 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
       -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
-      -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO"
+      -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO" \
+      -D LLVM_PARALLEL_COMPILE_JOBS=24
+
 
 echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.

>From 90fc649fdff74a9c11d1312b315eb9736ecee1f6 Mon Sep 17 00:00:00 2001
From: Lucile Nihlen <luci.the.rose at gmail.com>
Date: Fri, 24 May 2024 18:16:27 +0000
Subject: [PATCH 2/6] temp hardcode flang testing

---
 .ci/generate-buildkite-pipeline-premerge | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index e1c66ac18e7ac..8626de7bb4186 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -53,6 +53,7 @@ echo "Directories modified:" >&2
 echo "$modified_dirs" >&2
 
 function compute-projects-to-test() {
+  echo flang
   projects=${@}
   for project in ${projects}; do
     echo "${project}"

>From ac7c69554d024c88ba98cba0076cf598a3222fb8 Mon Sep 17 00:00:00 2001
From: Lucile Nihlen <luci.the.rose at gmail.com>
Date: Fri, 24 May 2024 18:36:49 +0000
Subject: [PATCH 3/6] remove temp flang test

---
 .ci/generate-buildkite-pipeline-premerge | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 8626de7bb4186..e1c66ac18e7ac 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -53,7 +53,6 @@ echo "Directories modified:" >&2
 echo "$modified_dirs" >&2
 
 function compute-projects-to-test() {
-  echo flang
   projects=${@}
   for project in ${projects}; do
     echo "${project}"

>From e4c0446194997562e408c4884fae3b68d78f7f2f Mon Sep 17 00:00:00 2001
From: Lucile Nihlen <luci.the.rose at gmail.com>
Date: Tue, 28 May 2024 14:31:33 +0000
Subject: [PATCH 4/6] revise numbers based on feedback

---
 .ci/monolithic-windows.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index d31a913826d7e..91e719c52d436 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -61,8 +61,8 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO" \
-      -D LLVM_PARALLEL_COMPILE_JOBS=24
-
+      -D LLVM_PARALLEL_COMPILE_JOBS=16 \
+      -D LLVM_PARALLEL_LINK_JOBS=4
 
 echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.

>From 064e3ff9c0ea84da8c7af0b06935f193b48f4aa9 Mon Sep 17 00:00:00 2001
From: Lucile Nihlen <luci.the.rose at gmail.com>
Date: Tue, 28 May 2024 15:16:17 +0000
Subject: [PATCH 5/6] force flang and clang to test

---
 .ci/generate-buildkite-pipeline-premerge | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index e1c66ac18e7ac..7343c051bd0bb 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -53,6 +53,7 @@ echo "Directories modified:" >&2
 echo "$modified_dirs" >&2
 
 function compute-projects-to-test() {
+  echo "flang clang"
   projects=${@}
   for project in ${projects}; do
     echo "${project}"

>From 4720c5116ab3a35f361321d3b2c87b4095004fcd Mon Sep 17 00:00:00 2001
From: Lucile Nihlen <luci.the.rose at gmail.com>
Date: Tue, 28 May 2024 19:52:22 +0000
Subject: [PATCH 6/6] revert testing hack

---
 .ci/generate-buildkite-pipeline-premerge | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 7343c051bd0bb..e1c66ac18e7ac 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -53,7 +53,6 @@ echo "Directories modified:" >&2
 echo "$modified_dirs" >&2
 
 function compute-projects-to-test() {
-  echo "flang clang"
   projects=${@}
   for project in ${projects}; do
     echo "${project}"