[llvm] [cross-project-tests] Derive tool substitutions from CMake and reject unlisted llvm-* tools (PR #225127)

[Alexander Richardson via llvm-commits]

https://github.com/arichardson updated https://github.com/llvm/llvm-project/pull/225127

>From 60d1d69e6371e2a81f7d64c9436be599ad4f758d Mon Sep 17 00:00:00 2001
From: Alex Richardson <alexrichardson at google.com>
Date: Mon, 21 Sep 2026 08:57:58 -0700
Subject: [PATCH] [cross-project-tests] Derive tool substitutions from CMake
 and reject unlisted llvm-* tools

Previously, `cross-project-tests/CMakeLists.txt` listed `llvm-*` binaries in
`CROSS_PROJECT_TEST_DEPS` without registering corresponding `ToolSubst` entries
in `cross-project-tests/lit.cfg.py`, so tests invoking tools such as
`llvm-objdump` or `llvm-dis` would silently pick them up from `$PATH` instead of
the build directory.

Factor the tool list into `CROSS_PROJECT_TEST_TOOLS` (adding `llvm-nm`,
`llvm-readelf`, and `llvm-readobj`), pass it through `lit.site.cfg.py.in` into
`lit.cfg.py` so the CMake dependencies and lit substitutions stay in sync, and
add a catch-all substitution mapping any remaining bare `llvm-*` command to
`false 'missing CROSS_PROJECT_TEST_TOOLS substitution for ...'` to catch missing
entries immediately.

This commit was created with the help of AI tools
---
 cross-project-tests/CMakeLists.txt     | 19 +++++++++++++------
 cross-project-tests/lit.cfg.py         | 10 ++++++++++
 cross-project-tests/lit.site.cfg.py.in |  1 +
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt
index 85102065e79e6..95187ef4e45be 100644
--- a/cross-project-tests/CMakeLists.txt
+++ b/cross-project-tests/CMakeLists.txt
@@ -17,23 +17,30 @@ add_subdirectory(debuginfo-tests/llvm-prettyprinters/lldb)
 set(CROSS_PROJECT_TESTS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(CROSS_PROJECT_TESTS_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
-set(CROSS_PROJECT_TEST_DEPS
-  FileCheck
-  check-gdb-llvm-support
-  ${LLDB_FORMATTER_TESTS}
-  count
+set(CROSS_PROJECT_TEST_TOOLS
   llvm-ar
   llvm-config
   llvm-dis
   llvm-dwarfdump
   llvm-lto2
   llvm-modextract
+  llvm-nm
   llvm-objdump
-  not
+  llvm-readelf
+  llvm-readobj
   obj2yaml
   split-file
   )
 
+set(CROSS_PROJECT_TEST_DEPS
+  FileCheck
+  check-gdb-llvm-support
+  ${LLDB_FORMATTER_TESTS}
+  count
+  not
+  ${CROSS_PROJECT_TEST_TOOLS}
+  )
+
 if ("clang" IN_LIST LLVM_ENABLE_PROJECTS)
   list(APPEND CROSS_PROJECT_TEST_DEPS clang)
 endif()
diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py
index 2df382615a0be..7f02e5ed6d9fd 100644
--- a/cross-project-tests/lit.cfg.py
+++ b/cross-project-tests/lit.cfg.py
@@ -57,6 +57,7 @@
     ),
     ToolSubst("%llvm_src_root", config.llvm_src_root),
     ToolSubst("%llvm_tools_dir", config.llvm_tools_dir),
+    *config.cross_project_test_tools,
 ]
 
 
@@ -292,6 +293,15 @@ def can_target_host():
 
 llvm_config.add_tool_substitutions(tools, tool_dirs)
 
+# Reject any unlisted llvm-* tool so tests cannot silently pick up binaries
+# from $PATH when a tool is missing from CROSS_PROJECT_TEST_TOOLS.
+config.substitutions.append(
+    (
+        r"(^|\s+|\|)\s*(llvm-[a-z0-9-]+)\b",
+        r"\1false 'missing CROSS_PROJECT_TEST_TOOLS substitution for \2'",
+    )
+)
+
 lit.util.usePlatformSdkOnDarwin(config, lit_config)
 
 
diff --git a/cross-project-tests/lit.site.cfg.py.in b/cross-project-tests/lit.site.cfg.py.in
index b8992b6dca45e..6d4c33592fa9b 100644
--- a/cross-project-tests/lit.site.cfg.py.in
+++ b/cross-project-tests/lit.site.cfg.py.in
@@ -17,6 +17,7 @@ config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.is_msvc = lit.util.pythonize_bool("@MSVC@")
 config.llvm_enabled_projects = "@LLVM_ENABLE_PROJECTS@".split(";")
+config.cross_project_test_tools = "@CROSS_PROJECT_TEST_TOOLS@".split(";")
 
 config.mlir_src_root = "@MLIR_SOURCE_DIR@"