[llvm] [CMake] Add tablegen job pool support (PR #84762)
David Stuttard via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 11 07:07:06 PDT 2024
https://github.com/dstutt created https://github.com/llvm/llvm-project/pull/84762
Add the ability to set the number of tablegen jobs that can run in parallel
similar to the LLVM_PARALLEL_[COMPILE|LINK]_JOBS options that already exist.
>From 8b99b815574de882da309be9ac0473d9c417b538 Mon Sep 17 00:00:00 2001
From: David Stuttard <david.stuttard at amd.com>
Date: Mon, 11 Mar 2024 11:21:28 +0000
Subject: [PATCH] [CMake] Add tablegen job pool support
Add the ability to set the number of tablegen jobs that can run in parallel
similar to the LLVM_PARALLEL_[COMPILE|LINK]_JOBS options that already exist.
---
llvm/cmake/modules/HandleLLVMOptions.cmake | 24 +++++++++++++++++++++-
llvm/cmake/modules/TableGen.cmake | 7 +++++++
llvm/docs/CMake.rst | 8 ++++++++
llvm/docs/GettingStarted.rst | 6 +++---
4 files changed, 41 insertions(+), 4 deletions(-)
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index eca2962cf82071..745a8354f11896 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -36,7 +36,7 @@ string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO)
# The following only works with the Ninja generator in CMake >= 3.0.
set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
"Define the maximum number of concurrent compilation jobs (Ninja only).")
-if(LLVM_RAM_PER_COMPILE_JOB OR LLVM_RAM_PER_LINK_JOB)
+if(LLVM_RAM_PER_COMPILE_JOB OR LLVM_RAM_PER_LINK_JOB OR LLVM_RAM_PER_TABLEGEN_JOB)
cmake_host_system_information(RESULT available_physical_memory QUERY AVAILABLE_PHYSICAL_MEMORY)
cmake_host_system_information(RESULT number_of_logical_cores QUERY NUMBER_OF_LOGICAL_CORES)
endif()
@@ -86,6 +86,28 @@ elseif(LLVM_PARALLEL_LINK_JOBS)
message(WARNING "Job pooling is only available with Ninja generators.")
endif()
+set(LLVM_PARALLEL_TABLEGEN_JOBS "" CACHE STRING
+ "Define the maximum number of concurrent tablegen jobs (Ninja only).")
+if(LLVM_RAM_PER_TABLEGEN_JOB)
+ math(EXPR jobs_with_sufficient_memory "${available_physical_memory} / ${LLVM_RAM_PER_TABLEGEN_JOB}" OUTPUT_FORMAT DECIMAL)
+ if (jobs_with_sufficient_memory LESS 1)
+ set(jobs_with_sufficient_memory 1)
+ endif()
+ if (jobs_with_sufficient_memory LESS number_of_logical_cores)
+ set(LLVM_PARALLEL_TABLEGEN_JOBS "${jobs_with_sufficient_memory}")
+ else()
+ set(LLVM_PARALLEL_TABLEGEN_JOBS "${number_of_logical_cores}")
+ endif()
+endif()
+if(LLVM_PARALLEL_TABLEGEN_JOBS)
+ if(NOT CMAKE_GENERATOR MATCHES "Ninja")
+ message(WARNING "Job pooling is only available with Ninja generators.")
+ else()
+ set_property(GLOBAL APPEND PROPERTY JOB_POOLS tablegen_job_pool=${LLVM_PARALLEL_TABLEGEN_JOBS})
+ # Job pool for tablegen is set on the add_custom_command
+ endif()
+endif()
+
if( LLVM_ENABLE_ASSERTIONS )
# MSVC doesn't like _DEBUG on release builds. See PR 4379.
if( NOT MSVC )
diff --git a/llvm/cmake/modules/TableGen.cmake b/llvm/cmake/modules/TableGen.cmake
index 1d18fdde2bb98b..df91598c404f57 100644
--- a/llvm/cmake/modules/TableGen.cmake
+++ b/llvm/cmake/modules/TableGen.cmake
@@ -125,6 +125,12 @@ function(tablegen project ofn)
set(tablegen_exe ${${project}_TABLEGEN_EXE})
set(tablegen_depends ${${project}_TABLEGEN_TARGET} ${tablegen_exe})
+ if(LLVM_PARALLEL_TABLEGEN_JOBS)
+ set(LLVM_TABLEGEN_JOB_POOL JOB_POOL tablegen_job_pool)
+ else()
+ set(LLVM_TABLEGEN_JOB_POOL "")
+ endif()
+
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
COMMAND ${tablegen_exe} ${ARG_UNPARSED_ARGUMENTS} -I ${CMAKE_CURRENT_SOURCE_DIR}
${tblgen_includes}
@@ -139,6 +145,7 @@ function(tablegen project ofn)
${local_tds} ${global_tds}
${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
${LLVM_TARGET_DEPENDS}
+ ${LLVM_TABLEGEN_JOB_POOL}
COMMENT "Building ${ofn}..."
)
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 35c47989a7eef0..2026a2e485c3a7 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -762,6 +762,9 @@ enabled sub-projects. Nearly all of these variable names begin with
**LLVM_PARALLEL_LINK_JOBS**:STRING
Define the maximum number of concurrent link jobs.
+**LLVM_PARALLEL_TABLEGEN_JOBS**:STRING
+ Define the maximum number of concurrent tablegen jobs.
+
**LLVM_PARALLEL_LIT**:BOOL
Defaults to ``OFF``. If set to ``OFF``, lit testsuites will be configured
with CMake's ``USES_TERMINAL`` flag to give direct access to the terminal. If
@@ -781,6 +784,11 @@ enabled sub-projects. Nearly all of these variable names begin with
to be sure its not terminated in your memory restricted environment. On ELF
platforms also consider ``LLVM_USE_SPLIT_DWARF`` in Debug build.
+**LLVM_RAM_PER_TABLEGEN_JOB**:STRING
+ Calculates the amount of Ninja tablegen jobs according to available resources.
+ Value has to be in MB, overwrites LLVM_PARALLEL_TABLEGEN_JOBS. Tablegen jobs
+ will be between one and amount of logical cores.
+
**LLVM_PROFDATA_FILE**:PATH
Path to a profdata file to pass into clang's -fprofile-instr-use flag. This
can only be specified if you're building with clang.
diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst
index 7634199babbad3..705f6427d9ed53 100644
--- a/llvm/docs/GettingStarted.rst
+++ b/llvm/docs/GettingStarted.rst
@@ -90,11 +90,11 @@ Getting the Source Code and Building LLVM
is installed on your system. This can dramatically speed up link times
if the default linker is slow.
- * ``-DLLVM_PARALLEL_{COMPILE,LINK}_JOBS=N`` --- Limit the number of
- compile/link jobs running in parallel at the same time. This is
+ * ``-DLLVM_PARALLEL_{COMPILE,LINK,TABLEGEN}_JOBS=N`` --- Limit the number of
+ compile/link/tablegen jobs running in parallel at the same time. This is
especially important for linking since linking can use lots of memory. If
you run into memory issues building LLVM, try setting this to limit the
- maximum number of compile/link jobs running at the same time.
+ maximum number of compile/link/tablegen jobs running at the same time.
* ``cmake --build build [--target <target>]`` or the build system specified
above directly.
More information about the llvm-commits
mailing list