[clang] fb80b6b - Make it possible for lit.site.cfg to contain relative paths, and use it for llvm and clang
Nico Weber via cfe-commits
cfe-commits at lists.llvm.org
Thu Apr 2 10:53:23 PDT 2020
Author: Nico Weber
Date: 2020-04-02T13:53:16-04:00
New Revision: fb80b6b2d58c476747a3206bd4371b787108591b
URL: https://github.com/llvm/llvm-project/commit/fb80b6b2d58c476747a3206bd4371b787108591b
DIFF: https://github.com/llvm/llvm-project/commit/fb80b6b2d58c476747a3206bd4371b787108591b.diff
LOG: Make it possible for lit.site.cfg to contain relative paths, and use it for llvm and clang
Currently, all generated lit.site.cfg files contain absolute paths.
This makes it impossible to build on one machine, and then transfer the
build output to another machine for test execution. Being able to do
this is useful for several use cases:
1. When running tests on an ARM machine, it would be possible to build
on a fast x86 machine and then copy build artifacts over after building.
2. It allows running several test suites (clang, llvm, lld) on 3
different machines, reducing test time from sum(each test suite time) to
max(each test suite time).
This patch makes it possible to pass a list of variables that should be
relative in the generated lit.site.cfg.py file to
configure_lit_site_cfg(). The lit.site.cfg.py.in file needs to call
`path()` on these variables, so that the paths are converted to absolute
form at lit start time.
The testers would have to have an LLVM checkout at the same revision,
and the build dir would have to be at the same relative path as on the
builder.
This does not yet cover how to figure out which files to copy from the
builder machine to the tester machines. (One idea is to look at the
`--graphviz=test.dot` output and copy all inputs of the `check-llvm`
target.)
Differential Revision: https://reviews.llvm.org/D77184
Added:
Modified:
clang/test/CMakeLists.txt
clang/test/Unit/lit.site.cfg.py.in
clang/test/lit.site.cfg.py.in
llvm/cmake/modules/AddLLVM.cmake
llvm/test/CMakeLists.txt
llvm/test/Unit/lit.site.cfg.py.in
llvm/test/lit.site.cfg.py.in
llvm/utils/gn/secondary/clang/test/BUILD.gn
llvm/utils/gn/secondary/llvm/test/BUILD.gn
Removed:
################################################################################
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index d453074d7926..ca326fea8066 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -35,6 +35,16 @@ configure_lit_site_cfg(
${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+ PATHS
+ "LLVM_SOURCE_DIR"
+ "LLVM_BINARY_DIR"
+ "LLVM_TOOLS_DIR"
+ "LLVM_LIBS_DIR"
+ "SHLIBDIR"
+ "LLVM_LIT_TOOLS_DIR"
+ "CLANG_BINARY_DIR"
+ "CLANG_SOURCE_DIR"
+ "CLANG_TOOLS_DIR"
)
configure_lit_site_cfg(
@@ -42,6 +52,13 @@ configure_lit_site_cfg(
${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
+ PATHS
+ "LLVM_SOURCE_DIR"
+ "LLVM_BINARY_DIR"
+ "LLVM_TOOLS_DIR"
+ "LLVM_LIBS_DIR"
+ "CLANG_BINARY_DIR"
+ "SHLIBDIR"
)
option(CLANG_TEST_USE_VG "Run Clang tests under Valgrind" OFF)
diff --git a/clang/test/Unit/lit.site.cfg.py.in b/clang/test/Unit/lit.site.cfg.py.in
index 715b4d93f58f..400a9c05e58c 100644
--- a/clang/test/Unit/lit.site.cfg.py.in
+++ b/clang/test/Unit/lit.site.cfg.py.in
@@ -2,14 +2,14 @@
import sys
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.llvm_src_root = path("@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path("@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path("@LLVM_TOOLS_DIR@")
+config.llvm_libs_dir = path("@LLVM_LIBS_DIR@")
config.llvm_build_mode = "@LLVM_BUILD_MODE@"
-config.clang_obj_root = "@CLANG_BINARY_DIR@"
+config.clang_obj_root = path("@CLANG_BINARY_DIR@")
config.enable_shared = @ENABLE_SHARED@
-config.shlibdir = "@SHLIBDIR@"
+config.shlibdir = path("@SHLIBDIR@")
config.target_triple = "@TARGET_TRIPLE@"
# Support substitution of the tools_dir, libs_dirs, and build_mode with user
@@ -25,4 +25,5 @@ except KeyError:
lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
# Let the main config do the real work.
-lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/Unit/lit.cfg.py")
+lit_config.load_config(
+ config, os.path.join(path("@CLANG_SOURCE_DIR@"), "test/Unit/lit.cfg.py"))
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index 62616d9a2b95..84dd0f8846cf 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -2,16 +2,16 @@
import sys
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
-config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_src_root = path("@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path("@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path("@LLVM_TOOLS_DIR@")
+config.llvm_libs_dir = path("@LLVM_LIBS_DIR@")
+config.llvm_shlib_dir = path("@SHLIBDIR@")
config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
-config.clang_obj_root = "@CLANG_BINARY_DIR@"
-config.clang_src_dir = "@CLANG_SOURCE_DIR@"
-config.clang_tools_dir = "@CLANG_TOOLS_DIR@"
+config.lit_tools_dir = path("@LLVM_LIT_TOOLS_DIR@")
+config.clang_obj_root = path("@CLANG_BINARY_DIR@")
+config.clang_src_dir = path("@CLANG_SOURCE_DIR@")
+config.clang_tools_dir = path("@CLANG_TOOLS_DIR@")
config.host_triple = "@LLVM_HOST_TRIPLE@"
config.target_triple = "@TARGET_TRIPLE@"
config.host_cxx = "@CMAKE_CXX_COMPILER@"
@@ -50,4 +50,5 @@ if not "@CLANG_DEFAULT_LINKER@":
config.available_features.add('platform-linker')
# Let the main config do the real work.
-lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/lit.cfg.py")
+lit_config.load_config(
+ config, os.path.join(config.clang_src_dir, "test/lit.cfg.py"))
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 238aa3eab9f4..74450b36523e 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -1396,8 +1396,14 @@ endmacro()
# variables needed for the 'lit.site.cfg' files. This function bundles the
# common variables that any Lit instance is likely to need, and custom
# variables can be passed in.
+# The keyword PATHS is followed by a list of cmake variable names that are
+# mentioned as `path("@varname@")` in the lit.cfg.py.in file. Variables in that
+# list are treated as paths that are relative to the directory the generated
+# lit.cfg.py file is in, and the `path()` function converts the relative
+# path back to absolute form. This makes it possible to move a build directory
+# containing lit.cfg.py files from one machine to another.
function(configure_lit_site_cfg site_in site_out)
- cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING" ${ARGN})
+ cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING;PATHS" ${ARGN})
if ("${ARG_MAIN_CONFIG}" STREQUAL "")
get_filename_component(INPUT_DIR ${site_in} DIRECTORY)
@@ -1447,12 +1453,17 @@ function(configure_lit_site_cfg site_in site_out)
set(HOST_CXX "${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER_ARG1}")
set(HOST_LDFLAGS "${CMAKE_EXE_LINKER_FLAGS}")
- set(LIT_SITE_CFG_IN_HEADER "## Autogenerated from ${site_in}\n## Do not edit!")
+ set(LIT_SITE_CFG_IN_HEADER "# Autogenerated from ${site_in}\n# Do not edit!")
+
+ string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
+ "# Allow generated lit.site.cfg.py to be relocatable.\n"
+ "def path(p): return os.path.join(os.path.dirname(__file__), p) if p else ''\n"
+ )
# Override config_target_triple (and the env)
if(LLVM_TARGET_TRIPLE_ENV)
# This is expanded into the heading.
- string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
+ string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}"
"import os\n"
"target_env = \"${LLVM_TARGET_TRIPLE_ENV}\"\n"
"config.target_triple = config.environment[target_env] = os.environ.get(target_env, \"${TARGET_TRIPLE}\")\n"
@@ -1462,7 +1473,45 @@ function(configure_lit_site_cfg site_in site_out)
set(TARGET_TRIPLE "\"+config.target_triple+\"")
endif()
+ if (ARG_PATH_VALUES)
+ # Walk ARG_PATHS and collect the current value of the variables in there.
+ foreach(path ${ARG_PATHS})
+ list(APPEND ARG_PATH_VALUES "${${path}}")
+ endforeach()
+
+ # Compute paths relative to the directory containing output lit.site.cfg.py.
+ # Passing ARG_PATH_VALUES as-is to execute_process() makes cmake strip
+ # empty list entries. So escape the ;s in the list and do the splitting
+ # outselves. cmake has no relpath function, so use Python for that.
+ string(REPLACE ";" "\\;" ARG_PATH_VALUES_ESCAPED "${ARG_PATH_VALUES}")
+ get_filename_component(OUTPUT_DIR ${site_out} DIRECTORY)
+ execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
+ "import os, sys; sys.stdout.write(';'.join(os.path.relpath(p, sys.argv[1]) if p else '' for p in sys.argv[2].split(';')))"
+ ${OUTPUT_DIR}
+ ${ARG_PATH_VALUES_ESCAPED}
+ OUTPUT_VARIABLE ARG_PATH_VALUES_RELATIVE)
+
+ list(LENGTH ARG_PATHS len_paths)
+ list(LENGTH ARG_PATH_VALUES len_path_values)
+ list(LENGTH ARG_PATH_VALUES_RELATIVE len_path_value_rels)
+ if ((NOT ${len_paths} EQUAL ${len_path_values}) OR
+ (NOT ${len_paths} EQUAL ${path_value_len_rels}))
+ message(SEND_ERROR "PATHS lengths got confused")
+ endif()
+
+ # Transform variables mentioned in ARG_PATHS to relative paths for
+ # the configure_file() call. Variables are copied to subscopeds by cmake,
+ # so this only modifies the local copy of the variables.
+ math(EXPR arg_path_limit "${len_paths} - 1")
+ foreach(i RANGE ${arg_path_limit})
+ list(GET ARG_PATHS ${i} val1)
+ list(GET ARG_PATH_VALUES_RELATIVE ${i} val2)
+ set(${val1} ${val2})
+ endforeach()
+ endif()
+
configure_file(${site_in} ${site_out} @ONLY)
+
if (EXISTS "${ARG_MAIN_CONFIG}")
set(PYTHON_STATEMENT "map_config('${ARG_MAIN_CONFIG}', '${site_out}')")
get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index d1bc970f3643..330a35212883 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -22,12 +22,23 @@ configure_lit_site_cfg(
${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+ PATHS
+ "LLVM_SOURCE_DIR"
+ "LLVM_BINARY_DIR"
+ "LLVM_TOOLS_DIR"
+ "LLVM_LIBRARY_DIR"
+ "SHLIBDIR"
)
configure_lit_site_cfg(
${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
+ PATHS
+ "LLVM_SOURCE_DIR"
+ "LLVM_BINARY_DIR"
+ "LLVM_TOOLS_DIR"
+ "SHLIBDIR"
)
# Set the depends list as a variable so that it can grow conditionally.
diff --git a/llvm/test/Unit/lit.site.cfg.py.in b/llvm/test/Unit/lit.site.cfg.py.in
index 1fef001be627..3358ccf89eed 100644
--- a/llvm/test/Unit/lit.site.cfg.py.in
+++ b/llvm/test/Unit/lit.site.cfg.py.in
@@ -2,12 +2,12 @@
import sys
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_src_root = path("@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path("@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path("@LLVM_TOOLS_DIR@")
config.llvm_build_mode = "@LLVM_BUILD_MODE@"
config.enable_shared = @ENABLE_SHARED@
-config.shlibdir = "@SHLIBDIR@"
+config.shlibdir = path("@SHLIBDIR@")
# Support substitution of the tools_dir and build_mode with user parameters.
# This is used when we can't determine the tool dir at configuration time.
@@ -20,4 +20,5 @@ except KeyError:
lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
# Let the main config do the real work.
-lit_config.load_config(config, "@LLVM_SOURCE_DIR@/test/Unit/lit.cfg.py")
+lit_config.load_config(
+ config, os.path.join(config.llvm_src_root, "test/Unit/lit.cfg.py"))
diff --git a/llvm/test/lit.site.cfg.py.in b/llvm/test/lit.site.cfg.py.in
index 6f4d5f790828..2af0ec336089 100644
--- a/llvm/test/lit.site.cfg.py.in
+++ b/llvm/test/lit.site.cfg.py.in
@@ -4,14 +4,14 @@ import sys
config.host_triple = "@LLVM_HOST_TRIPLE@"
config.target_triple = "@TARGET_TRIPLE@"
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
-config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_src_root = path("@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path("@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path("@LLVM_TOOLS_DIR@")
+config.llvm_lib_dir = path("@LLVM_LIBRARY_DIR@")
+config.llvm_shlib_dir = path("@SHLIBDIR@")
config.llvm_shlib_ext = "@SHLIBEXT@"
config.llvm_exe_ext = "@EXEEXT@"
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.lit_tools_dir = path("@LLVM_LIT_TOOLS_DIR@")
config.python_executable = "@PYTHON_EXECUTABLE@"
config.gold_executable = "@GOLD_EXECUTABLE@"
config.ld64_executable = "@LD64_EXECUTABLE@"
@@ -63,4 +63,5 @@ import lit.llvm
lit.llvm.initialize(lit_config, config)
# Let the main config do the real work.
-lit_config.load_config(config, "@LLVM_SOURCE_DIR@/test/lit.cfg.py")
+lit_config.load_config(
+ config, os.path.join(config.llvm_src_root, "test/lit.cfg.py"))
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 874891e89c81..84ac2e893cdf 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -12,7 +12,10 @@ template("write_lit_config") {
input = invoker.input
output = invoker.output
values = [
- "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+ # FIXME: Write relative paths for path()s.
+ "LIT_SITE_CFG_IN_HEADER=" +
+ "## Autogenerated from $input, do not edit\n\n" +
+ "def path(p): return p if p else \"\"",
"CLANG_BINARY_DIR=" +
rebase_path(get_label_info("//clang", "target_out_dir")),
"CLANG_SOURCE_DIR=" + rebase_path("//clang"),
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 0f7d90d1ab60..c7740ac4cc99 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -14,7 +14,10 @@ template("write_lit_config") {
input = invoker.input
output = invoker.output
values = [
- "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+ # FIXME: Write relative paths for path()s.
+ "LIT_SITE_CFG_IN_HEADER=" +
+ "## Autogenerated from $input, do not edit\n\n" +
+ "def path(p): return p if p else \"\"",
"ENABLE_SHARED=0",
"LLVM_BINARY_DIR=" +
rebase_path(get_label_info("//llvm", "target_out_dir")),
More information about the cfe-commits
mailing list