[llvm] 6655c53 - [cmake] Serialize native builds for Make generator (#121021)

[via llvm-commits]

Author: arthurqiu
Date: 2025-01-15T11:42:21-08:00
New Revision: 6655c53ff02f0ffca0fcaa3261e14c2a6417d3da

URL: https://github.com/llvm/llvm-project/commit/6655c53ff02f0ffca0fcaa3261e14c2a6417d3da
DIFF: https://github.com/llvm/llvm-project/commit/6655c53ff02f0ffca0fcaa3261e14c2a6417d3da.diff

LOG: [cmake] Serialize native builds for Make generator (#121021)

The build system is fragile by allowing multiple invocation of
subprocess builds in the native folder for Make generator.

For example, during sub-invocation of the native llvm-config,
llvm-min-tblgen is also built. If there is another sub-invocation of the
native llvm-min-tblgen build running in parallel, they may overwrite
each other's build results, and may lead to errors like "Text file
busy".

This patch adds a cmake script that uses file lock to serialize all
native builds for Make generator.

Added: 
    llvm/cmake/modules/FileLock.cmake

Modified: 
    llvm/cmake/modules/CrossCompile.cmake
    llvm/cmake/modules/LLVMExternalProjectUtils.cmake

Removed: 
    


################################################################################
diff  --git a/llvm/cmake/modules/CrossCompile.cmake b/llvm/cmake/modules/CrossCompile.cmake
index c2cc84bec15217..3b31d3e218a37f 100644
--- a/llvm/cmake/modules/CrossCompile.cmake
+++ b/llvm/cmake/modules/CrossCompile.cmake
@@ -12,6 +12,14 @@ function(llvm_create_cross_target project_name target_name toolchain buildtype)
     message(STATUS "Setting native build dir to " ${${project_name}_${target_name}_BUILD})
   endif(NOT DEFINED ${project_name}_${target_name}_BUILD)
 
+  if(NOT DEFINED ${project_name}_${target_name}_STAMP)
+    set(${project_name}_${target_name}_STAMP
+      "${CMAKE_CURRENT_BINARY_DIR}/${target_name}-stamps")
+    set(${project_name}_${target_name}_STAMP
+      ${${project_name}_${target_name}_STAMP} PARENT_SCOPE)
+    message(STATUS "Setting native stamp dir to " ${${project_name}_${target_name}_STAMP})
+  endif(NOT DEFINED ${project_name}_${target_name}_STAMP)
+
   if (EXISTS ${LLVM_MAIN_SRC_DIR}/cmake/platforms/${toolchain}.cmake)
     set(CROSS_TOOLCHAIN_FLAGS_INIT
       -DCMAKE_TOOLCHAIN_FILE=\"${LLVM_MAIN_SRC_DIR}/cmake/platforms/${toolchain}.cmake\")
@@ -130,13 +138,16 @@ function(build_native_tool target output_path_var)
     set_property(GLOBAL APPEND PROPERTY ${PROJECT_NAME}_HOST_TARGETS ${output_path})
   endif()
 
-  llvm_ExternalProject_BuildCmd(build_cmd ${target} ${${PROJECT_NAME}_NATIVE_BUILD}
+  llvm_ExternalProject_BuildCmd(build_cmd ${target}
+                                ${${PROJECT_NAME}_NATIVE_BUILD}
+                                ${${PROJECT_NAME}_NATIVE_STAMP}
                                 CONFIGURATION Release)
   add_custom_command(OUTPUT "${output_path}"
                      COMMAND ${build_cmd}
                      DEPENDS CONFIGURE_${PROJECT_NAME}_NATIVE ${ARG_DEPENDS} ${host_targets}
                      WORKING_DIRECTORY "${${PROJECT_NAME}_NATIVE_BUILD}"
                      COMMENT "Building native ${target}..."
-                     USES_TERMINAL)
+                     USES_TERMINAL
+                     VERBATIM)
   set(${output_path_var} "${output_path}" PARENT_SCOPE)
 endfunction()

diff  --git a/llvm/cmake/modules/FileLock.cmake b/llvm/cmake/modules/FileLock.cmake
new file mode 100644
index 00000000000000..1e403a625847e3
--- /dev/null
+++ b/llvm/cmake/modules/FileLock.cmake
@@ -0,0 +1,9 @@
+# CMake script that synchronizes process execution on a given file lock.
+#
+# Input variables:
+#   LOCK_FILE_PATH    - The file to be locked for the scope of the process of this cmake script.
+#   COMMAND           - The command to be executed.
+
+file(LOCK ${LOCK_FILE_PATH})
+string(REPLACE "@" ";" command_args ${COMMAND})
+execute_process(COMMAND ${command_args} COMMAND_ERROR_IS_FATAL ANY)

diff  --git a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
index f7e1165bc4b947..55422c2a4c023d 100644
--- a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
+++ b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
@@ -2,14 +2,18 @@ include(ExternalProject)
 
 # llvm_ExternalProject_BuildCmd(out_var target)
 #   Utility function for constructing command lines for external project targets
-function(llvm_ExternalProject_BuildCmd out_var target bin_dir)
+function(llvm_ExternalProject_BuildCmd out_var target bin_dir stamp_dir)
   cmake_parse_arguments(ARG "" "CONFIGURATION" "" ${ARGN})
   if(NOT ARG_CONFIGURATION)
     set(ARG_CONFIGURATION "$<CONFIG>")
   endif()
   if (CMAKE_GENERATOR MATCHES "Make")
     # Use special command for Makefiles to support parallelism.
-    set(${out_var} "$(MAKE)" "-C" "${bin_dir}" "${target}" PARENT_SCOPE)
+    string(JOIN "@" make_cmd "$(MAKE)" "-C" "${bin_dir}" "${target}")
+    set(file_lock_script "${LLVM_CMAKE_DIR}/FileLock.cmake")
+    set(${out_var} ${CMAKE_COMMAND} "-DLOCK_FILE_PATH=${stamp_dir}/cmake.lock"
+                                    "-DCOMMAND=${make_cmd}"
+                                    "-P" "${file_lock_script}" PARENT_SCOPE)
   else()
     set(tool_args "${LLVM_EXTERNAL_PROJECT_BUILD_TOOL_ARGS}")
     if(NOT tool_args STREQUAL "")
@@ -409,7 +413,7 @@ function(llvm_ExternalProject_Add name source_dir)
     set(force_deps DEPENDS ${TOOLCHAIN_BINS})
   endif()
 
-  llvm_ExternalProject_BuildCmd(run_clean clean ${BINARY_DIR})
+  llvm_ExternalProject_BuildCmd(run_clean clean ${BINARY_DIR} ${STAMP_DIR})
   ExternalProject_Add_Step(${name} clean
     COMMAND ${run_clean}
     COMMENT "Cleaning ${name}..."
@@ -449,7 +453,7 @@ function(llvm_ExternalProject_Add name source_dir)
     else()
       set(external_target "${target}")
     endif()
-    llvm_ExternalProject_BuildCmd(build_runtime_cmd ${external_target} ${BINARY_DIR})
+    llvm_ExternalProject_BuildCmd(build_runtime_cmd ${external_target} ${BINARY_DIR} ${STAMP_DIR})
     add_custom_target(${target}
       COMMAND ${build_runtime_cmd}
       DEPENDS ${name}-configure