[llvm] [cmake] Option to create Ninja job pools depending on available resources (PR #65274)

[via llvm-commits]

https://github.com/jusito created https://github.com/llvm/llvm-project/pull/65274:

This PR adds options to let CMake calculate the ninja job pools depending on free memory and available cores.
You can provide memory requirements for each compile and link job which is checked against CMake AVAILABLE_PHYSICAL_MEMORY and NUMBER_OF_LOGICAL_CORES. [This information are available since CMake 3.0](https://cmake.org/cmake/help/v3.0/command/cmake_host_system_information.html).

This is very helpful in CI environments with multiple jobs per environment or a VM with multiple users.
Its different to LLVM_PARALLEL_LINK_JOBS / LLVM_PARALLEL_COMPILE_JOBS (or ninja -j 1) because it tries to use the resources more efficient without being terminated. Only downside currently is that compile and link jobs can run at the same time so there is an offset for link job memory suggested which is added to the documentation.

The definitions aren't added as cache because if I understand it correctly this would break it because values could be outdated.

>From 37d5ef40327ada2c7e0917aad74e65e6ddb618fd Mon Sep 17 00:00:00 2001
From: Lucas Briese <lucas.briese at iem.fraunhofer.de>
Date: Mon, 4 Sep 2023 17:49:42 +0200
Subject: [PATCH] [cmake] Option to create Ninja job pools depending on
 available resources.

Dynamically couple compile time to available resources in memory constrained environments.
Tweaked alternative to LLVM_PARALLEL_{COMPILE,LINK}_JOBS=1.
---
 llvm/cmake/modules/HandleLLVMOptions.cmake | 25 ++++++++++++++++++++++
 llvm/docs/CMake.rst                        | 13 +++++++++++
 2 files changed, 38 insertions(+)

diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 29b242d80051f0d..64ca598f4a0c7c9 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -36,6 +36,20 @@ string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO)
 # The following only works with the Ninja generator in CMake >= 3.0.
 set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
   "Define the maximum number of concurrent compilation jobs (Ninja only).")
+
+cmake_host_system_information(RESULT available_physical_memory QUERY AVAILABLE_PHYSICAL_MEMORY)
+cmake_host_system_information(RESULT number_of_logical_cores QUERY NUMBER_OF_LOGICAL_CORES)
+if(LLVM_RAM_PER_COMPILE_JOB)
+  math(EXPR memory_available_jobs "${available_physical_memory} / ${LLVM_RAM_PER_COMPILE_JOB}" OUTPUT_FORMAT DECIMAL)
+  if (memory_available_jobs LESS 1)
+    set(memory_available_jobs 1)
+  endif()
+  if (memory_available_jobs LESS number_of_logical_cores)
+    set(LLVM_PARALLEL_COMPILE_JOBS "${memory_available_jobs}")
+  else()
+    set(LLVM_PARALLEL_COMPILE_JOBS "${number_of_logical_cores}")
+  endif()
+endif()
 if(LLVM_PARALLEL_COMPILE_JOBS)
   if(NOT CMAKE_GENERATOR MATCHES "Ninja")
     message(WARNING "Job pooling is only available with Ninja generators.")
@@ -47,6 +61,17 @@ endif()
 
 set(LLVM_PARALLEL_LINK_JOBS "" CACHE STRING
   "Define the maximum number of concurrent link jobs (Ninja only).")
+if(LLVM_RAM_PER_LINK_JOB)
+  math(EXPR memory_available_jobs "${available_physical_memory} / ${LLVM_RAM_PER_LINK_JOB}" OUTPUT_FORMAT DECIMAL)
+  if (memory_available_jobs LESS 1)
+    set(memory_available_jobs 1)
+  endif()
+  if (memory_available_jobs LESS number_of_logical_cores)
+    set(LLVM_PARALLEL_LINK_JOBS "${memory_available_jobs}")
+  else()
+    set(LLVM_PARALLEL_LINK_JOBS "${number_of_logical_cores}")
+  endif()
+endif()
 if(CMAKE_GENERATOR MATCHES "Ninja")
   if(NOT LLVM_PARALLEL_LINK_JOBS AND uppercase_LLVM_ENABLE_LTO STREQUAL "THIN")
     message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.")
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 67f740447ed7bfb..6e61806a2b85c9a 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -736,6 +736,19 @@ enabled sub-projects. Nearly all of these variable names begin with
 **LLVM_PARALLEL_LINK_JOBS**:STRING
   Define the maximum number of concurrent link jobs.
 
+**LLVM_RAM_PER_COMPILE_JOB**:STRING
+  Calculates the amount of Ninja compile jobs according to available resources.
+  Value has to be in MB, overwrites LLVM_PARALLEL_COMPILE_JOBS. Compile jobs 
+  will be between one and amount of logical cores.
+
+**LLVM_RAM_PER_LINK_JOB**:STRING
+  Calculates the amount of Ninja link jobs according to available resources.
+  Value has to be in MB, overwrites LLVM_PARALLEL_LINK_JOBS. Link jobs will 
+  be between one and amount of logical cores. Link jobs will not run 
+  exclusively therefore you should add an offset of one or two compile jobs 
+  to be sure its not terminated in your memory restricted environment. On ELF
+  platforms also consider ``LLVM_USE_SPLIT_DWARF`` in Debug build.
+
 **LLVM_PROFDATA_FILE**:PATH
   Path to a profdata file to pass into clang's -fprofile-instr-use flag. This
   can only be specified if you're building with clang.