[libcxx] r230592 - Add remote testing support to the lit config.
Jonathan Roelofs
jonathan at codesourcery.com
Wed Feb 25 16:42:18 PST 2015
Author: jroelofs
Date: Wed Feb 25 18:42:17 2015
New Revision: 230592
URL: http://llvm.org/viewvc/llvm-project?rev=230592&view=rev
Log:
Add remote testing support to the lit config.
Executors can be specified at configure time by using the -DLIBCXX_EXECUTOR=""
option. Examples include:
$ cmake <other_flags> -DLIBCXX_EXECUTOR="TimeoutExecutor(30,LocalExecutor())"
This runs individual tests with a maximum duration
$ cmake <other_flags> -DLIBCXX_EXECUTOR="SSHExecutor('hostname','username')"
This runs tests on a remote target, using scp to shuttle binaries to the
target, and ssh to invoke commands there.
$ cmake <other_flags> -DLIBCXX_EXECUTOR="PrefixExecutor('/path/to/run/script',LocalExecutor())"
This assumes the script knows how to copy run the executables passed to it,
and allows for the ultimate control. This is useful for running things
inside emulators like Valgrind & QEMU.
TODO: This doesn't claim to support ShTest tests yet, that will take a bit more
thought & finagling (I'm still not sure how to orchestrate copy-in for those cases.
I've also punted on what to do about tests that read data files. The testsuite
has several tests that need to read *.dat files placed next to them, and
currently those aren't copied over when using, say, an SSHExecutor. The
affected tests are:
libc++ :: std/input.output/file.streams/fstreams/filebuf.virtuals/pbackfail.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/filebuf.virtuals/underflow.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/member_swap.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/move_assign.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/nonmember_swap.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/move.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/pointer.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/string.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.members/close.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.members/open_pointer.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.members/open_string.pass.cpp
libc++ :: std/input.output/file.streams/fstreams/ifstream.members/rdbuf.pass.cpp
libc++ :: std/localization/locales/locale.convenience/conversions/conversions.buffer/pbackfail.pass.cpp
libc++ :: std/localization/locales/locale.convenience/conversions/conversions.buffer/underflow.pass.cpp
Note: One thing to watch out for when using the SSHExecutor for cross-testing is
that you'll also want to specify a TargetInfo object (so that the host's
features aren't used for available-features checks and flags setup).
http://reviews.llvm.org/D7380
Added:
libcxx/trunk/test/libcxx/test/executor.py
libcxx/trunk/test/libcxx/test/tracing.py
Modified:
libcxx/trunk/test/CMakeLists.txt
libcxx/trunk/test/libcxx/test/config.py
libcxx/trunk/test/libcxx/test/format.py
libcxx/trunk/test/lit.site.cfg.in
Modified: libcxx/trunk/test/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/CMakeLists.txt?rev=230592&r1=230591&r2=230592&view=diff
==============================================================================
--- libcxx/trunk/test/CMakeLists.txt (original)
+++ libcxx/trunk/test/CMakeLists.txt Wed Feb 25 18:42:17 2015
@@ -46,6 +46,8 @@ if (LIT_EXECUTABLE)
pythonize_bool(LIBCXX_ENABLE_MONOTONIC_CLOCK)
set(LIBCXX_TARGET_INFO "libcxx.test.target_info.LocalTI" CACHE STRING
"TargetInfo to use when setting up test environment.")
+ set(LIBCXX_EXECUTOR "None" CACHE STRING
+ "Executor to use when running tests.")
set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!")
Modified: libcxx/trunk/test/libcxx/test/config.py
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/libcxx/test/config.py?rev=230592&r1=230591&r2=230592&view=diff
==============================================================================
--- libcxx/trunk/test/libcxx/test/config.py (original)
+++ libcxx/trunk/test/libcxx/test/config.py Wed Feb 25 18:42:17 2015
@@ -12,7 +12,8 @@ import lit.util # pylint: disable=impor
from libcxx.test.format import LibcxxTestFormat
from libcxx.compiler import CXXCompiler
-
+from libcxx.test.executor import *
+from libcxx.test.tracing import *
def loadSiteConfig(lit_config, config, param_name, env_name):
# We haven't loaded the site specific configuration (the user is
@@ -78,6 +79,7 @@ class Configuration(object):
"parameter '{}' should be true or false".format(name))
def configure(self):
+ self.configure_executor()
self.configure_target_info()
self.configure_cxx()
self.configure_triple()
@@ -110,6 +112,32 @@ class Configuration(object):
list(self.config.available_features))
self.lit_config.note('Using environment: %r' % self.env)
+ def get_test_format(self):
+ return LibcxxTestFormat(
+ self.cxx,
+ self.use_clang_verify,
+ self.execute_external,
+ self.executor,
+ exec_env=self.env)
+
+ def configure_executor(self):
+ exec_str = self.get_lit_conf('executor', "None")
+ te = eval(exec_str)
+ if te:
+ self.lit_config.note("Using executor: %r" % exec_str)
+ if self.lit_config.useValgrind:
+ # We have no way of knowing where in the chain the
+ # ValgrindExecutor is supposed to go. It is likely
+ # that the user wants it at the end, but we have no
+ # way of getting at that easily.
+ selt.lit_config.fatal("Cannot infer how to create a Valgrind "
+ " executor.")
+ else:
+ te = LocalExecutor()
+ if self.lit_config.useValgrind:
+ te = ValgrindExecutor(self.lit_config.valgrindArgs, te)
+ self.executor = te
+
def configure_target_info(self):
default = "libcxx.test.target_info.LocalTI"
info_str = self.get_lit_conf('target_info', default)
@@ -119,13 +147,6 @@ class Configuration(object):
if info_str != default:
self.lit_config.note("inferred target_info as: %r" % info_str)
- def get_test_format(self):
- return LibcxxTestFormat(
- self.cxx,
- self.use_clang_verify,
- self.execute_external,
- exec_env=self.env)
-
def configure_cxx(self):
# Gather various compiler parameters.
cxx = self.get_lit_conf('cxx_under_test')
@@ -319,7 +340,7 @@ class Configuration(object):
# Configure include paths
self.cxx.compile_flags += ['-nostdinc++']
self.configure_compile_flags_header_includes()
- if sys.platform.startswith('linux'):
+ if self.target_info.platform() == 'linux':
self.cxx.compile_flags += ['-D__STDC_FORMAT_MACROS',
'-D__STDC_LIMIT_MACROS',
'-D__STDC_CONSTANT_MACROS']
@@ -601,15 +622,16 @@ class Configuration(object):
# linux-gnu is needed in the triple to properly identify linuxes
# that use GLIBC. Handle redhat and opensuse triples as special
# cases and append the missing `-gnu` portion.
- if target_triple.endswith('redhat-linux') or \
- target_triple.endswith('suse-linux'):
+ if (target_triple.endswith('redhat-linux') or
+ target_triple.endswith('suse-linux')):
target_triple += '-gnu'
self.config.target_triple = target_triple
self.lit_config.note(
"inferred target_triple as: %r" % self.config.target_triple)
def configure_env(self):
- if sys.platform == 'darwin' and not self.use_system_cxx_lib:
+ if (self.target_info.platform() == 'darwin' and
+ not self.use_system_cxx_lib):
libcxx_library = self.get_lit_conf('libcxx_library')
if libcxx_library:
cxx_library_root = os.path.dirname(libcxx_library)
Added: libcxx/trunk/test/libcxx/test/executor.py
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/libcxx/test/executor.py?rev=230592&view=auto
==============================================================================
--- libcxx/trunk/test/libcxx/test/executor.py (added)
+++ libcxx/trunk/test/libcxx/test/executor.py Wed Feb 25 18:42:17 2015
@@ -0,0 +1,166 @@
+import os
+
+import tracing
+
+from lit.util import executeCommand # pylint: disable=import-error
+
+
+class Executor(object):
+ def run(self, exe_path, cmd, local_cwd, env=None):
+ """Execute a command.
+ Be very careful not to change shared state in this function.
+ Executor objects are shared between python processes in `lit -jN`.
+ Args:
+ exe_path: str: Local path to the executable to be run
+ cmd: [str]: subprocess.call style command
+ local_cwd: str: Local path to the working directory
+ env: {str: str}: Environment variables to execute under
+ Returns:
+ out, err, exitCode
+ """
+ raise NotImplementedError
+
+
+class LocalExecutor(Executor):
+ def __init__(self):
+ super(LocalExecutor, self).__init__()
+
+ def run(self, exe_path, cmd=None, work_dir='.', env=None):
+ cmd = cmd or [exe_path]
+ env_cmd = []
+ if env:
+ env_cmd += ['env']
+ env_cmd += ['%s=%s' % (k, v) for k, v in env.items()]
+ if work_dir == '.':
+ work_dir = os.getcwd()
+ return executeCommand(env_cmd + cmd, cwd=work_dir)
+
+
+class PrefixExecutor(Executor):
+ """Prefix an executor with some other command wrapper.
+
+ Most useful for setting ulimits on commands, or running an emulator like
+ qemu and valgrind.
+ """
+ def __init__(self, commandPrefix, chain):
+ super(PrefixExecutor, self).__init__()
+
+ self.commandPrefix = commandPrefix
+ self.chain = chain
+
+ def run(self, exe_path, cmd=None, work_dir='.', env=None):
+ cmd = cmd or [exe_path]
+ return self.chain.run(self.commandPrefix + cmd, work_dir, env=env)
+
+
+class PostfixExecutor(Executor):
+ """Postfix an executor with some args."""
+ def __init__(self, commandPostfix, chain):
+ super(PostfixExecutor, self).__init__()
+
+ self.commandPostfix = commandPostfix
+ self.chain = chain
+
+ def run(self, exe_path, cmd=None, work_dir='.', env=None):
+ cmd = cmd or [exe_path]
+ return self.chain.run(cmd + self.commandPostfix, work_dir, env=env)
+
+
+
+class TimeoutExecutor(PrefixExecutor):
+ """Execute another action under a timeout.
+
+ Deprecated. http://reviews.llvm.org/D6584 adds timeouts to LIT.
+ """
+ def __init__(self, duration, chain):
+ super(TimeoutExecutor, self).__init__(
+ ['timeout', duration], chain)
+
+
+class SSHExecutor(Executor):
+ def __init__(self, host, username=None):
+ super(SSHExecutor, self).__init__()
+
+ self.user_prefix = username + '@' if username else ''
+ self.host = host
+ self.scp_command = 'scp'
+ self.ssh_command = 'ssh'
+
+ self.local_run = executeCommand
+ # TODO(jroelofs): switch this on some -super-verbose-debug config flag
+ if False:
+ self.local_run = tracing.trace_function(
+ self.local_run, log_calls=True, log_results=True,
+ label='ssh_local')
+
+ def remote_temp_dir(self):
+ return self._remote_temp(True)
+
+ def remote_temp_file(self):
+ return self._remote_temp(False)
+
+ def _remote_temp(self, is_dir):
+ # TODO: detect what the target system is, and use the correct
+ # mktemp command for it. (linux and darwin differ here, and I'm
+ # sure windows has another way to do it)
+
+ # Not sure how to do suffix on osx yet
+ dir_arg = '-d' if is_dir else ''
+ cmd = 'mktemp -q {} /tmp/libcxx.XXXXXXXXXX'.format(dir_arg)
+ temp_path, err, exitCode = self.__execute_command_remote([cmd])
+ temp_path = temp_path.strip()
+ if exitCode != 0:
+ raise RuntimeError(err)
+ return temp_path
+
+ def copy_in(self, local_srcs, remote_dsts):
+ scp = self.scp_command
+ remote = self.host
+ remote = self.user_prefix + remote
+
+ # This could be wrapped up in a tar->scp->untar for performance
+ # if there are lots of files to be copied/moved
+ for src, dst in zip(local_srcs, remote_dsts):
+ cmd = [scp, '-p', src, remote + ':' + dst]
+ self.local_run(cmd)
+
+ def delete_remote(self, remote):
+ try:
+ self.__execute_command_remote(['rm', '-rf', remote])
+ except OSError:
+ # TODO: Log failure to delete?
+ pass
+
+ def run(self, exe_path, cmd=None, work_dir='.', env=None):
+ target_exe_path = None
+ target_cwd = None
+ try:
+ target_exe_path = self.remote_temp_file()
+ target_cwd = self.remote_temp_dir()
+ if cmd:
+ # Replace exe_path with target_exe_path.
+ cmd = [c if c != exe_path else target_exe_path for c in cmd]
+ else:
+ cmd = [target_exe_path]
+ self.copy_in([exe_path], [target_exe_path])
+ return self.__execute_command_remote(cmd, target_cwd, env)
+ except:
+ raise
+ finally:
+ if target_exe_path:
+ self.delete_remote(target_exe_path)
+ if target_cwd:
+ self.delete_remote(target_cwd)
+
+ def __execute_command_remote(self, cmd, remote_work_dir='.', env=None):
+ remote = self.user_prefix + self.host
+ ssh_cmd = [self.ssh_command, '-oBatchMode=yes', remote]
+ if env:
+ env_cmd = ['env'] + ['%s=%s' % (k, v) for k, v in env.items()]
+ else:
+ env_cmd = []
+ remote_cmd = ' '.join(env_cmd + cmd)
+ if remote_work_dir != '.':
+ remote_cmd = 'cd ' + remote_work_dir + ' && ' + remote_cmd
+ return self.local_run(ssh_cmd + [remote_cmd])
+
Modified: libcxx/trunk/test/libcxx/test/format.py
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/libcxx/test/format.py?rev=230592&r1=230591&r2=230592&view=diff
==============================================================================
--- libcxx/trunk/test/libcxx/test/format.py (original)
+++ libcxx/trunk/test/libcxx/test/format.py Wed Feb 25 18:42:17 2015
@@ -20,10 +20,12 @@ class LibcxxTestFormat(object):
FOO.sh.cpp - A test that uses LIT's ShTest format.
"""
- def __init__(self, cxx, use_verify_for_fail, execute_external, exec_env):
+ def __init__(self, cxx, use_verify_for_fail, execute_external,
+ executor, exec_env):
self.cxx = cxx
self.use_verify_for_fail = use_verify_for_fail
self.execute_external = execute_external
+ self.executor = executor
self.exec_env = dict(exec_env)
# TODO: Move this into lit's FileBasedTest
@@ -73,6 +75,10 @@ class LibcxxTestFormat(object):
# Dispatch the test based on its suffix.
if is_sh_test:
+ if self.executor:
+ # We can't run ShTest tests with a executor yet.
+ # For now, bail on trying to run them
+ return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
return lit.TestRunner._runShTest(test, lit_config,
self.execute_external, script,
tmpBase, execDir)
@@ -104,15 +110,12 @@ class LibcxxTestFormat(object):
report += "Compilation failed unexpectedly!"
return lit.Test.FAIL, report
# Run the test
- cmd = []
+ local_cwd = os.path.dirname(source_path)
+ env = None
if self.exec_env:
- cmd += ['env']
- cmd += ['%s=%s' % (k, v) for k, v in self.exec_env.items()]
- if lit_config.useValgrind:
- cmd = lit_config.valgrindArgs + cmd
- cmd += [exec_path]
- out, err, rc = lit.util.executeCommand(
- cmd, cwd=os.path.dirname(source_path))
+ env = self.exec_env
+ out, err, rc = self.executor.run(exec_path, [exec_path],
+ local_cwd, env)
if rc != 0:
report = libcxx.util.makeReport(cmd, out, err, rc)
report = "Compiled With: %s\n%s" % (compile_cmd, report)
Added: libcxx/trunk/test/libcxx/test/tracing.py
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/libcxx/test/tracing.py?rev=230592&view=auto
==============================================================================
--- libcxx/trunk/test/libcxx/test/tracing.py (added)
+++ libcxx/trunk/test/libcxx/test/tracing.py Wed Feb 25 18:42:17 2015
@@ -0,0 +1,34 @@
+import os
+import inspect
+
+
+def trace_function(function, log_calls, log_results, label=''):
+ def wrapper(*args, **kwargs):
+ kwarg_strs = ['{}={}'.format(k, v) for (k, v) in kwargs]
+ arg_str = ', '.join([str(a) for a in args] + kwarg_strs)
+ call_str = '{}({})'.format(function.func_name, arg_str)
+
+ # Perform the call itself, logging before, after, and anything thrown.
+ try:
+ if log_calls:
+ print '{}: Calling {}'.format(label, call_str)
+ res = function(*args, **kwargs)
+ if log_results:
+ print '{}: {} -> {}'.format(label, call_str, res)
+ return res
+ except Exception as ex:
+ if log_results:
+ print '{}: {} raised {}'.format(label, call_str, type(ex))
+ raise ex
+
+ return wrapper
+
+
+def trace_object(obj, log_calls, log_results, label=''):
+ for name, member in inspect.getmembers(obj):
+ if inspect.ismethod(member):
+ # Skip meta-functions, decorate everything else
+ if not member.func_name.startswith('__'):
+ setattr(obj, name, trace_function(member, log_calls,
+ log_results, label))
+ return obj
Modified: libcxx/trunk/test/lit.site.cfg.in
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/lit.site.cfg.in?rev=230592&r1=230591&r2=230592&view=diff
==============================================================================
--- libcxx/trunk/test/lit.site.cfg.in (original)
+++ libcxx/trunk/test/lit.site.cfg.in Wed Feb 25 18:42:17 2015
@@ -18,6 +18,7 @@ config.target_triple = "@LIBC
config.sysroot = "@LIBCXX_SYSROOT@"
config.gcc_toolchain = "@LIBCXX_GCC_TOOLCHAIN@"
config.target_info = "@LIBCXX_TARGET_INFO@"
+config.executor = "@LIBCXX_EXECUTOR@"
# Let the main config do the real work.
lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg")
More information about the cfe-commits
mailing list