[Lldb-commits] [lldb] 32e90bb - [lldb][test] Support remote run of Shell tests (#95986)
via lldb-commits
lldb-commits at lists.llvm.org
Mon Oct 7 11:31:36 PDT 2024
Author: Vladislav Dzhidzhoev
Date: 2024-10-07T20:31:33+02:00
New Revision: 32e90bbe579d39356ebd269158b1b5eef82761d4
URL: https://github.com/llvm/llvm-project/commit/32e90bbe579d39356ebd269158b1b5eef82761d4
DIFF: https://github.com/llvm/llvm-project/commit/32e90bbe579d39356ebd269158b1b5eef82761d4.diff
LOG: [lldb][test] Support remote run of Shell tests (#95986)
1. This commit adds LLDB_TEST_PLATFORM_URL, LLDB_TEST_SYSROOT,
LLDB_TEST_PLATFORM_WORKING_DIR, LLDB_SHELL_TESTS_DISABLE_REMOTE cmake
flags to pass arguments for cross-compilation and remote running of both Shell&API tests.
2. To run Shell tests remotely, it adds 'platform select' and 'platform connect' commands to %lldb
substitution.
3. 'remote-linux' feature added to lit to disable tests failing with
remote execution.
4. A separate working directory is assigned to each test to avoid
conflicts during parallel test execution.
5. Remote Shell testing is run only when LLDB_TEST_SYSROOT is set for
building test sources. The recommended compiler for that is Clang.
---------
Co-authored-by: Vladimir Vereschaka <vvereschaka at accesssoftek.com>
Added:
lldb/test/Shell/Settings/TestEchoCommandsQuiet.test
Modified:
lldb/docs/resources/test.rst
lldb/test/API/lit.cfg.py
lldb/test/API/lit.site.cfg.py.in
lldb/test/CMakeLists.txt
lldb/test/Shell/Expr/TestIRMemoryMap.test
lldb/test/Shell/Settings/TestEchoCommands.test
lldb/test/Shell/Target/target-label.test
lldb/test/Shell/helper/toolchain.py
lldb/test/Shell/lit.cfg.py
lldb/test/Shell/lit.site.cfg.py.in
Removed:
################################################################################
diff --git a/lldb/docs/resources/test.rst b/lldb/docs/resources/test.rst
index 906e687cb13b13..715d3772fe2787 100644
--- a/lldb/docs/resources/test.rst
+++ b/lldb/docs/resources/test.rst
@@ -60,7 +60,8 @@ something like ``target.BreakpointCreateByName`` [#]_.
A good rule of thumb is to prefer shell tests when what is being tested is
relatively simple. Expressivity is limited compared to the API tests, which
means that you have to have a well-defined test scenario that you can easily
-match with ``FileCheck``.
+match with ``FileCheck``. Though Shell tests can be run remotely, behavior
+specific to remote debugging must be tested with API tests instead.
Another thing to consider are the binaries being debugged, which we call
inferiors. For shell tests, they have to be relatively simple. The
@@ -592,15 +593,17 @@ test suite, but there are two things to have in mind:
multiple connections. For more information on how to setup remote debugging
see the Remote debugging page.
2. You must tell the test-suite how to connect to the remote system. This is
- achieved using the ``--platform-name``, ``--platform-url`` and
- ``--platform-working-dir`` parameters to ``dotest.py``. These parameters
- correspond to the platform select and platform connect LLDB commands. You
- will usually also need to specify the compiler and architecture for the
- remote system.
-
-Currently, running the remote test suite is supported only with ``dotest.py`` (or
-dosep.py with a single thread), but we expect this issue to be addressed in the
-near future.
+ achieved using the ``LLDB_TEST_PLATFORM_URL``, ``LLDB_TEST_PLATFORM_WORKING_DIR``
+ flags to cmake, and ``--platform-name`` parameter to ``dotest.py``.
+ These parameters correspond to the platform select and platform connect
+ LLDB commands. You will usually also need to specify the compiler and
+ architecture for the remote system.
+3. Remote Shell tests execution is currently supported only for Linux target
+ platform. It's triggered when ``LLDB_TEST_SYSROOT`` is provided for building
+ test sources. It can be disabled by setting ``LLDB_TEST_SHELL_DISABLE_REMOTE=On``.
+ Shell tests are not guaranteed to pass against remote target if the compiler
+ being used is other than Clang.
+
Running tests in QEMU System Emulation Environment
``````````````````````````````````````````````````
diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index 96520c7c826246..6a0a1b0a766755 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -303,6 +303,13 @@ def delete_module_cache(path):
# In particular, (1) is visited at the top of the file, since the script
# derives other information from it.
+if is_configured("lldb_platform_url"):
+ dotest_cmd += ["--platform-url", config.lldb_platform_url]
+if is_configured("lldb_platform_working_dir"):
+ dotest_cmd += ["--platform-working-dir", config.lldb_platform_working_dir]
+if is_configured("cmake_sysroot"):
+ dotest_cmd += ["--sysroot", config.cmake_sysroot]
+
if is_configured("dotest_user_args_str"):
dotest_cmd.extend(config.dotest_user_args_str.split(";"))
diff --git a/lldb/test/API/lit.site.cfg.py.in b/lldb/test/API/lit.site.cfg.py.in
index 8b2d09ae41cd2a..db3cd2971f347a 100644
--- a/lldb/test/API/lit.site.cfg.py.in
+++ b/lldb/test/API/lit.site.cfg.py.in
@@ -24,6 +24,9 @@ config.lua_executable = "@Lua_EXECUTABLE@"
config.lua_test_entry = "TestLuaAPI.py"
config.dotest_common_args_str = lit_config.substitute("@LLDB_TEST_COMMON_ARGS@")
config.dotest_user_args_str = lit_config.substitute("@LLDB_TEST_USER_ARGS@")
+config.lldb_platform_url = lit_config.substitute("@LLDB_TEST_PLATFORM_URL@")
+config.lldb_platform_working_dir = lit_config.substitute("@LLDB_TEST_PLATFORM_WORKING_DIR@")
+config.cmake_sysroot = lit_config.substitute("@LLDB_TEST_SYSROOT@" or "@DEFAULT_SYSROOT@")
config.lldb_enable_python = @LLDB_ENABLE_PYTHON@
config.dotest_lit_args_str = None
config.enabled_plugins = []
diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt
index 5ac474736eb63d..25037934f636c7 100644
--- a/lldb/test/CMakeLists.txt
+++ b/lldb/test/CMakeLists.txt
@@ -235,6 +235,8 @@ if (CMAKE_SIZEOF_VOID_P EQUAL 8)
set(LLDB_IS_64_BITS 1)
endif()
+set(LLDB_TEST_SHELL_DISABLE_REMOTE OFF CACHE BOOL "Disable remote Shell tests execution")
+
# These values are not canonicalized within LLVM.
llvm_canonicalize_cmake_booleans(
LLDB_BUILD_INTEL_PT
@@ -244,6 +246,7 @@ llvm_canonicalize_cmake_booleans(
LLVM_ENABLE_ZLIB
LLVM_ENABLE_SHARED_LIBS
LLDB_HAS_LIBCXX
+ LLDB_TEST_SHELL_DISABLE_REMOTE
LLDB_TOOL_LLDB_SERVER_BUILD
LLDB_USE_SYSTEM_DEBUGSERVER
LLDB_IS_64_BITS)
diff --git a/lldb/test/Shell/Expr/TestIRMemoryMap.test b/lldb/test/Shell/Expr/TestIRMemoryMap.test
index 9dd0413be14cf0..9cba0624c92688 100644
--- a/lldb/test/Shell/Expr/TestIRMemoryMap.test
+++ b/lldb/test/Shell/Expr/TestIRMemoryMap.test
@@ -1,4 +1,4 @@
-# UNSUPPORTED: system-windows
+# UNSUPPORTED: system-windows, remote-linux
# RUN: %clangxx_host %p/Inputs/call-function.cpp -g -o %t
diff --git a/lldb/test/Shell/Settings/TestEchoCommands.test b/lldb/test/Shell/Settings/TestEchoCommands.test
index 234b9742bfa2aa..3488e707321863 100644
--- a/lldb/test/Shell/Settings/TestEchoCommands.test
+++ b/lldb/test/Shell/Settings/TestEchoCommands.test
@@ -1,8 +1,3 @@
# RUN: %lldb -x -b -o 'settings set interpreter.echo-comment-commands true' -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsAll.out
# RUN: %lldb -x -b -o 'settings set interpreter.echo-comment-commands false' -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsNoComments.out
# RUN: %lldb -x -b -o 'settings set interpreter.echo-commands false' -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsNone.out
-
-RUN: echo start >%t.file
-RUN: %lldb -x -b --source-quietly -s %S/Inputs/EchoCommandsTest.in >>%t.file
-RUN: echo done >>%t.file
-RUN: FileCheck %S/Inputs/EchoCommandsQuiet.out <%t.file
diff --git a/lldb/test/Shell/Settings/TestEchoCommandsQuiet.test b/lldb/test/Shell/Settings/TestEchoCommandsQuiet.test
new file mode 100644
index 00000000000000..88c4f598e69748
--- /dev/null
+++ b/lldb/test/Shell/Settings/TestEchoCommandsQuiet.test
@@ -0,0 +1,6 @@
+Can't mute platform select command.
+UNSUPPORTED: remote-linux
+RUN: echo start >%t.file
+RUN: %lldb -x -b --source-quietly -s %S/Inputs/EchoCommandsTest.in >>%t.file
+RUN: echo done >>%t.file
+RUN: FileCheck %S/Inputs/EchoCommandsQuiet.out <%t.file
diff --git a/lldb/test/Shell/Target/target-label.test b/lldb/test/Shell/Target/target-label.test
index 5ac430601e29ac..7f4f31e09fa164 100644
--- a/lldb/test/Shell/Target/target-label.test
+++ b/lldb/test/Shell/Target/target-label.test
@@ -4,16 +4,16 @@
target create -l "ls" /bin/ls
target list
-# CHECK: * target #0 (ls): /bin/ls
+# CHECK: * target #0 (ls): [[LS_PATH:.*]]
script lldb.target.SetLabel("")
target list
-# CHECK: * target #0: /bin/ls
+# CHECK: * target #0: [[LS_PATH]]
target create -l "cat" /bin/cat
target list
-# CHECK: target #0: /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0: [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH:.*]]
target create -l "cat" /bin/cat
# CHECK: Cannot use label 'cat' since it's set in target #1.
@@ -22,12 +22,12 @@ target create -l 42 /bin/cat
# CHECK: error: Cannot use integer as target label.
target select 0
-# CHECK: * target #0: /bin/ls
-# CHECK-NEXT: target #1 (cat): /bin/cat
+# CHECK: * target #0: [[LS_PATH]]
+# CHECK-NEXT: target #1 (cat): [[CAT_PATH]]
target select cat
-# CHECK: target #0: /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0: [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH]]
script lldb.target.GetLabel()
# CHECK: 'cat'
@@ -36,5 +36,5 @@ script lldb.debugger.GetTargetAtIndex(0).SetLabel('Not cat')
# CHECK: success
target list
-# CHECK: target #0 (Not cat): /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0 (Not cat): [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH]]
diff --git a/lldb/test/Shell/helper/toolchain.py b/lldb/test/Shell/helper/toolchain.py
index 255955fc70d8c4..81ae490f6a7dc4 100644
--- a/lldb/test/Shell/helper/toolchain.py
+++ b/lldb/test/Shell/helper/toolchain.py
@@ -1,14 +1,17 @@
import os
import itertools
import platform
+import re
import subprocess
import sys
import lit.util
+from lit.formats import ShTest
from lit.llvm import llvm_config
from lit.llvm.subst import FindTool
from lit.llvm.subst import ToolSubst
+import posixpath
def _get_lldb_init_path(config):
return os.path.join(config.test_exec_root, "lit-lldb-init-quiet")
@@ -22,6 +25,68 @@ def _disallow(config, execName):
config.substitutions.append((" {0} ".format(execName), warning.format(execName)))
+def get_lldb_args(config, suffix=""):
+ lldb_args = []
+ if "remote-linux" in config.available_features:
+ lldb_args += [
+ "-O",
+ '"platform select remote-linux"',
+ "-O",
+ f'"platform connect {config.lldb_platform_url}"',
+ ]
+ if config.lldb_platform_working_dir:
+ dir = posixpath.join(f"{config.lldb_platform_working_dir}", "shell")
+ if suffix:
+ dir += posixpath.join(dir, f"{suffix}")
+ lldb_args += [
+ "-O",
+ f'"platform shell mkdir -p {dir}"',
+ "-O",
+ f'"platform settings -w {dir}"',
+ ]
+ lldb_args += ["--no-lldbinit", "-S", _get_lldb_init_path(config)]
+ return lldb_args
+
+
+class ShTestLldb(ShTest):
+ def __init__(
+ self, execute_external=False, extra_substitutions=[], preamble_commands=[]
+ ):
+ super().__init__(execute_external, extra_substitutions, preamble_commands)
+
+ def execute(self, test, litConfig):
+ # Run each Shell test in a separate directory (on remote).
+
+ # Find directory change command in %lldb substitution.
+ for i, t in enumerate(test.config.substitutions):
+ if re.match(t[0], "%lldb"):
+ cmd = t[1]
+ if '-O "platform settings -w ' in cmd:
+ # If command is present, it is added by get_lldb_args.
+ # Replace the path with the tests' path in suite.
+ # Example:
+ # bin/lldb
+ # -O "platform shell mkdir -p /home/user/shell"
+ # -O "platform settings -w /home/user/shell" ...
+ # =>
+ # bin/lldb
+ # -O "platform shell mkdir -p /home/user/shell/SymbolFile/Breakpad/inline-record.test"
+ # -O "platform settings -w /home/user/shell/SymbolFile/Breakpad/inline-record.test" ...
+ args_def = " ".join(get_lldb_args(test.config))
+ args_unique = " ".join(
+ get_lldb_args(
+ test.config,
+ posixpath.join(*test.path_in_suite),
+ )
+ )
+ test.config.substitutions[i] = (
+ t[0],
+ cmd.replace(args_def, args_unique),
+ )
+ break
+ return super().execute(test, litConfig)
+
+
def use_lldb_substitutions(config):
# Set up substitutions for primary tools. These tools must come from config.lldb_tools_dir
# which is basically the build output directory. We do not want to find these in path or
@@ -34,7 +99,9 @@ def use_lldb_substitutions(config):
build_script = os.path.join(build_script, "build.py")
build_script_args = [
build_script,
- "--compiler=any", # Default to best compiler
+ (
+ "--compiler=clang" if config.enable_remote else "--compiler=any"
+ ), # Default to best compiler
"--arch=" + str(config.lldb_bitness),
]
if config.lldb_lit_tools_dir:
@@ -56,7 +123,7 @@ def use_lldb_substitutions(config):
ToolSubst(
"%lldb",
command=FindTool("lldb"),
- extra_args=["--no-lldbinit", "-S", lldb_init],
+ extra_args=get_lldb_args(config),
unresolved="fatal",
),
ToolSubst(
@@ -138,7 +205,10 @@ def use_support_substitutions(config):
# Set up substitutions for support tools. These tools can be overridden at the CMake
# level (by specifying -DLLDB_LIT_TOOLS_DIR), installed, or as a last resort, we can use
# the just-built version.
- host_flags = ["--target=" + config.host_triple]
+ if config.enable_remote:
+ host_flags = ["--target=" + config.target_triple]
+ else:
+ host_flags = ["--target=" + config.host_triple]
if platform.system() in ["Darwin"]:
try:
out = subprocess.check_output(["xcrun", "--show-sdk-path"]).strip()
@@ -165,6 +235,12 @@ def use_support_substitutions(config):
if config.cmake_sysroot:
host_flags += ["--sysroot={}".format(config.cmake_sysroot)]
+ if config.enable_remote and config.has_libcxx:
+ host_flags += [
+ "-L{}".format(config.libcxx_libs_dir),
+ "-lc++",
+ ]
+
host_flags = " ".join(host_flags)
config.substitutions.append(("%clang_host", "%clang " + host_flags))
config.substitutions.append(("%clangxx_host", "%clangxx " + host_flags))
diff --git a/lldb/test/Shell/lit.cfg.py b/lldb/test/Shell/lit.cfg.py
index d764cfa20ea837..9a7b71889f8dff 100644
--- a/lldb/test/Shell/lit.cfg.py
+++ b/lldb/test/Shell/lit.cfg.py
@@ -21,7 +21,7 @@
config.name = "lldb-shell"
# testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.test_format = toolchain.ShTestLldb(not llvm_config.use_lit_shell)
# suffixes: A list of file extensions to treat as test files. This is overriden
# by individual lit.local.cfg files in the test subdirectories.
@@ -68,6 +68,13 @@
lit_config.note("Running Shell tests in {} mode.".format(lldb_repro_mode))
toolchain.use_lldb_repro_substitutions(config, lldb_repro_mode)
+if config.lldb_platform_url and config.cmake_sysroot and config.enable_remote:
+ if re.match(r".*-linux.*", config.target_triple):
+ config.available_features.add("remote-linux")
+else:
+ # After this, enable_remote == True iff remote testing is going to be used.
+ config.enable_remote = False
+
llvm_config.use_default_substitutions()
toolchain.use_lldb_substitutions(config)
toolchain.use_support_substitutions(config)
diff --git a/lldb/test/Shell/lit.site.cfg.py.in b/lldb/test/Shell/lit.site.cfg.py.in
index b69e7bce1bc0be..31a6d68618b770 100644
--- a/lldb/test/Shell/lit.site.cfg.py.in
+++ b/lldb/test/Shell/lit.site.cfg.py.in
@@ -10,10 +10,15 @@ config.lldb_src_root = "@LLDB_SOURCE_DIR@"
config.lldb_obj_root = "@LLDB_BINARY_DIR@"
config.lldb_libs_dir = lit_config.substitute("@LLDB_LIBS_DIR@")
config.lldb_tools_dir = lit_config.substitute("@LLDB_TOOLS_DIR@")
+config.lldb_platform_url = lit_config.substitute("@LLDB_TEST_PLATFORM_URL@")
+config.lldb_platform_working_dir = lit_config.substitute("@LLDB_TEST_PLATFORM_WORKING_DIR@")
# Since it comes from the command line, it may have backslashes which
# should not need to be escaped.
config.lldb_lit_tools_dir = lit_config.substitute(r"@LLDB_LIT_TOOLS_DIR@")
-config.cmake_sysroot = lit_config.substitute("@CMAKE_SYSROOT@")
+config.cmake_sysroot = lit_config.substitute("@LLDB_TEST_SYSROOT@" or "@DEFAULT_SYSROOT@")
+config.has_libcxx = @LLDB_HAS_LIBCXX@
+config.enable_remote = not @LLDB_TEST_SHELL_DISABLE_REMOTE@
+config.libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
config.target_triple = "@LLVM_TARGET_TRIPLE@"
config.python_executable = "@Python3_EXECUTABLE@"
config.have_zlib = @LLVM_ENABLE_ZLIB@
More information about the lldb-commits
mailing list