[llvm] 9228859 - [CMake] Add tablegen job pool support (#84762)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 12 03:07:06 PDT 2024
Author: David Stuttard
Date: 2024-03-12T10:07:02Z
New Revision: 9228859c2a5aed307dc61edb4cfd6bee7b4c5949
URL: https://github.com/llvm/llvm-project/commit/9228859c2a5aed307dc61edb4cfd6bee7b4c5949
DIFF: https://github.com/llvm/llvm-project/commit/9228859c2a5aed307dc61edb4cfd6bee7b4c5949.diff
LOG: [CMake] Add tablegen job pool support (#84762)
Add the ability to set the number of tablegen jobs that can run in
parallel
similar to the LLVM_PARALLEL_[COMPILE|LINK]_JOBS options that already
exist.
Added:
Modified:
llvm/cmake/modules/HandleLLVMOptions.cmake
llvm/cmake/modules/TableGen.cmake
llvm/docs/CMake.rst
llvm/docs/GettingStarted.rst
Removed:
################################################################################
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index eca2962cf82071..745a8354f11896 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -36,7 +36,7 @@ string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO)
# The following only works with the Ninja generator in CMake >= 3.0.
set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
"Define the maximum number of concurrent compilation jobs (Ninja only).")
-if(LLVM_RAM_PER_COMPILE_JOB OR LLVM_RAM_PER_LINK_JOB)
+if(LLVM_RAM_PER_COMPILE_JOB OR LLVM_RAM_PER_LINK_JOB OR LLVM_RAM_PER_TABLEGEN_JOB)
cmake_host_system_information(RESULT available_physical_memory QUERY AVAILABLE_PHYSICAL_MEMORY)
cmake_host_system_information(RESULT number_of_logical_cores QUERY NUMBER_OF_LOGICAL_CORES)
endif()
@@ -86,6 +86,28 @@ elseif(LLVM_PARALLEL_LINK_JOBS)
message(WARNING "Job pooling is only available with Ninja generators.")
endif()
+set(LLVM_PARALLEL_TABLEGEN_JOBS "" CACHE STRING
+ "Define the maximum number of concurrent tablegen jobs (Ninja only).")
+if(LLVM_RAM_PER_TABLEGEN_JOB)
+ math(EXPR jobs_with_sufficient_memory "${available_physical_memory} / ${LLVM_RAM_PER_TABLEGEN_JOB}" OUTPUT_FORMAT DECIMAL)
+ if (jobs_with_sufficient_memory LESS 1)
+ set(jobs_with_sufficient_memory 1)
+ endif()
+ if (jobs_with_sufficient_memory LESS number_of_logical_cores)
+ set(LLVM_PARALLEL_TABLEGEN_JOBS "${jobs_with_sufficient_memory}")
+ else()
+ set(LLVM_PARALLEL_TABLEGEN_JOBS "${number_of_logical_cores}")
+ endif()
+endif()
+if(LLVM_PARALLEL_TABLEGEN_JOBS)
+ if(NOT CMAKE_GENERATOR MATCHES "Ninja")
+ message(WARNING "Job pooling is only available with Ninja generators.")
+ else()
+ set_property(GLOBAL APPEND PROPERTY JOB_POOLS tablegen_job_pool=${LLVM_PARALLEL_TABLEGEN_JOBS})
+ # Job pool for tablegen is set on the add_custom_command
+ endif()
+endif()
+
if( LLVM_ENABLE_ASSERTIONS )
# MSVC doesn't like _DEBUG on release builds. See PR 4379.
if( NOT MSVC )
diff --git a/llvm/cmake/modules/TableGen.cmake b/llvm/cmake/modules/TableGen.cmake
index 1d18fdde2bb98b..df91598c404f57 100644
--- a/llvm/cmake/modules/TableGen.cmake
+++ b/llvm/cmake/modules/TableGen.cmake
@@ -125,6 +125,12 @@ function(tablegen project ofn)
set(tablegen_exe ${${project}_TABLEGEN_EXE})
set(tablegen_depends ${${project}_TABLEGEN_TARGET} ${tablegen_exe})
+ if(LLVM_PARALLEL_TABLEGEN_JOBS)
+ set(LLVM_TABLEGEN_JOB_POOL JOB_POOL tablegen_job_pool)
+ else()
+ set(LLVM_TABLEGEN_JOB_POOL "")
+ endif()
+
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
COMMAND ${tablegen_exe} ${ARG_UNPARSED_ARGUMENTS} -I ${CMAKE_CURRENT_SOURCE_DIR}
${tblgen_includes}
@@ -139,6 +145,7 @@ function(tablegen project ofn)
${local_tds} ${global_tds}
${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
${LLVM_TARGET_DEPENDS}
+ ${LLVM_TABLEGEN_JOB_POOL}
COMMENT "Building ${ofn}..."
)
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 1490b38feb1eb5..d2f66d71d39afb 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -762,6 +762,9 @@ enabled sub-projects. Nearly all of these variable names begin with
**LLVM_PARALLEL_LINK_JOBS**:STRING
Define the maximum number of concurrent link jobs.
+**LLVM_PARALLEL_TABLEGEN_JOBS**:STRING
+ Define the maximum number of concurrent tablegen jobs.
+
**LLVM_RAM_PER_COMPILE_JOB**:STRING
Calculates the amount of Ninja compile jobs according to available resources.
Value has to be in MB, overwrites LLVM_PARALLEL_COMPILE_JOBS. Compile jobs
@@ -775,6 +778,11 @@ enabled sub-projects. Nearly all of these variable names begin with
to be sure its not terminated in your memory restricted environment. On ELF
platforms also consider ``LLVM_USE_SPLIT_DWARF`` in Debug build.
+**LLVM_RAM_PER_TABLEGEN_JOB**:STRING
+ Calculates the amount of Ninja tablegen jobs according to available resources.
+ Value has to be in MB, overwrites LLVM_PARALLEL_TABLEGEN_JOBS. Tablegen jobs
+ will be between one and amount of logical cores.
+
**LLVM_PROFDATA_FILE**:PATH
Path to a profdata file to pass into clang's -fprofile-instr-use flag. This
can only be specified if you're building with clang.
diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst
index 7634199babbad3..705f6427d9ed53 100644
--- a/llvm/docs/GettingStarted.rst
+++ b/llvm/docs/GettingStarted.rst
@@ -90,11 +90,11 @@ Getting the Source Code and Building LLVM
is installed on your system. This can dramatically speed up link times
if the default linker is slow.
- * ``-DLLVM_PARALLEL_{COMPILE,LINK}_JOBS=N`` --- Limit the number of
- compile/link jobs running in parallel at the same time. This is
+ * ``-DLLVM_PARALLEL_{COMPILE,LINK,TABLEGEN}_JOBS=N`` --- Limit the number of
+ compile/link/tablegen jobs running in parallel at the same time. This is
especially important for linking since linking can use lots of memory. If
you run into memory issues building LLVM, try setting this to limit the
- maximum number of compile/link jobs running at the same time.
+ maximum number of compile/link/tablegen jobs running at the same time.
* ``cmake --build build [--target <target>]`` or the build system specified
above directly.
More information about the llvm-commits
mailing list