[Lldb-commits] [lldb] r246794 - Roll dosep.py parallel test runner into dotest.py command line
Todd Fiala via lldb-commits
lldb-commits at lists.llvm.org
Thu Sep 3 11:58:45 PDT 2015
Author: tfiala
Date: Thu Sep 3 13:58:44 2015
New Revision: 246794
URL: http://llvm.org/viewvc/llvm-project?rev=246794&view=rev
Log:
Roll dosep.py parallel test runner into dotest.py command line
See the following for details:
http://reviews.llvm.org/D12587
Modified:
lldb/trunk/test/CMakeLists.txt
lldb/trunk/test/Makefile
lldb/trunk/test/dosep.py
lldb/trunk/test/dotest.py
lldb/trunk/test/dotest_args.py
lldb/trunk/www/test.html
Modified: lldb/trunk/test/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/CMakeLists.txt?rev=246794&r1=246793&r2=246794&view=diff
==============================================================================
--- lldb/trunk/test/CMakeLists.txt (original)
+++ lldb/trunk/test/CMakeLists.txt Thu Sep 3 13:58:44 2015
@@ -62,16 +62,16 @@ endif()
add_python_test_target(check-lldb-single
${LLDB_SOURCE_DIR}/test/dotest.py
- "${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
+ "--no-multiprocess;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
"Testing LLDB with args: ${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}"
)
-set(LLDB_DOSEP_ARGS -o;\"-q;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}\")
+set(LLDB_DOTEST_ARGS -q;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS})
# If tests crash cause LLDB to crash, or things are otherwise unstable, or if machine-parsable
# output is desired (i.e. in continuous integration contexts) check-lldb-single is a better target.
add_python_test_target(check-lldb
- ${LLDB_SOURCE_DIR}/test/dosep.py
- "${LLDB_DOSEP_ARGS}"
- "Testing LLDB (with a separate subprocess per test)"
+ ${LLDB_SOURCE_DIR}/test/dotest.py
+ "${LLDB_DOTEST_ARGS}"
+ "Testing LLDB (parallel execution, with a separate subprocess per test)"
)
Modified: lldb/trunk/test/Makefile
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/Makefile?rev=246794&r1=246793&r2=246794&view=diff
==============================================================================
--- lldb/trunk/test/Makefile (original)
+++ lldb/trunk/test/Makefile Thu Sep 3 13:58:44 2015
@@ -30,4 +30,4 @@ clean::
#----------------------------------------------------------------------
check-local::
rm -rf lldb-test-traces
- python $(PROJ_SRC_DIR)/dosep.py -o "--executable $(ToolDir)/lldb -q -s lldb-test-traces -u CXXFLAGS -u CFLAGS -C $(subst ccache,,$(CC))"
+ python $(PROJ_SRC_DIR)/dotest.py --executable $(ToolDir)/lldb -q -s lldb-test-traces -u CXXFLAGS -u CFLAGS -C $(subst ccache,,$(CC))
Modified: lldb/trunk/test/dosep.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/dosep.py?rev=246794&r1=246793&r2=246794&view=diff
==============================================================================
--- lldb/trunk/test/dosep.py (original)
+++ lldb/trunk/test/dosep.py Thu Sep 3 13:58:44 2015
@@ -38,7 +38,6 @@ import fnmatch
import platform
import re
import dotest_args
-import shlex
import subprocess
import sys
@@ -131,8 +130,6 @@ def parse_test_results(output):
result, re.MULTILINE)
unexpected_success_count = re.search("^RESULT:.*([0-9]+) unexpected successes",
result, re.MULTILINE)
- this_fail_count = 0
- this_error_count = 0
if pass_count is not None:
passes = passes + int(pass_count.group(1))
if fail_count is not None:
@@ -183,7 +180,7 @@ def process_dir(root, files, test_root,
script_file = os.path.join(test_root, "dotest.py")
command = ([sys.executable, script_file] +
dotest_argv +
- ["-p", name, root])
+ ["--inferior", "-p", name, root])
timeout_name = os.path.basename(os.path.splitext(name)[0]).upper()
@@ -201,7 +198,7 @@ def process_dir(root, files, test_root,
if status != ePassed]
unexpected_passes = [name for name, _, _, _, unexpected_successes in results
if unexpected_successes > 0]
-
+
pass_count = sum([result[2] for result in results])
fail_count = sum([result[3] for result in results])
@@ -284,7 +281,6 @@ def getExpectedTimeouts(platform_name):
else:
m = re.search('remote-(\w+)', platform_name)
target = m.group(1)
- remote = True
expected_timeout = set()
@@ -358,7 +354,27 @@ def find(pattern, path):
return result
-def main():
+def main(print_details_on_success, num_threads, test_subdir):
+ """Run dotest.py in inferior mode in parallel.
+
+ @param print_details_on_success the parsed value of the output-on-success
+ command line argument. When True, details of a successful dotest inferior
+ are printed even when everything succeeds. The normal behavior is to
+ not print any details when all the inferior tests pass.
+
+ @param num_threads the parsed value of the num-threads command line
+ argument.
+
+ @param test_subdir optionally specifies a subdir to limit testing
+ within. May be None if the entire test tree is to be used. This subdir
+ is assumed to be relative to the lldb/test root of the test hierarchy.
+ """
+
+ dotest_argv = sys.argv[1:]
+
+ global output_on_success
+ output_on_success = print_details_on_success
+
# We can't use sys.path[0] to determine the script directory
# because it doesn't work under a debugger
test_directory = os.path.dirname(os.path.realpath(__file__))
@@ -382,37 +398,8 @@ Run lldb test suite using a separate pro
E.g., export LLDB_TEST_TIMEOUT=0
or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0
""")
- parser.add_option(
- '-o', '--options',
- type='string', action='store',
- dest='dotest_options',
- help="""The options passed to 'dotest.py' if specified.""")
-
- parser.add_option(
- '-s', '--output-on-success',
- action='store_true',
- dest='output_on_success',
- default=False,
- help="""Print full output of 'dotest.py' even when it succeeds.""")
-
- parser.add_option(
- '-t', '--threads',
- type='int',
- dest='num_threads',
- help="""The number of threads to use when running tests separately.""")
-
- opts, args = parser.parse_args()
- dotest_option_string = opts.dotest_options
-
- is_posix = (os.name == "posix")
- dotest_argv = (shlex.split(dotest_option_string, posix=is_posix)
- if dotest_option_string
- else [])
-
parser = dotest_args.create_parser()
global dotest_options
- global output_on_success
- output_on_success = opts.output_on_success
dotest_options = dotest_args.parse_args(parser, dotest_argv)
if not dotest_options.s:
@@ -428,19 +415,17 @@ Run lldb test suite using a separate pro
session_dir = os.path.join(os.getcwd(), dotest_options.s)
# The root directory was specified on the command line
- if len(args) == 0:
- test_subdir = test_directory
+ if test_subdir and len(test_subdir) > 0:
+ test_subdir = os.path.join(test_directory, test_subdir)
else:
- test_subdir = os.path.join(test_directory, args[0])
+ test_subdir = test_directory
# clean core files in test tree from previous runs (Linux)
cores = find('core.*', test_subdir)
for core in cores:
os.unlink(core)
- if opts.num_threads:
- num_threads = opts.num_threads
- else:
+ if not num_threads:
num_threads_str = os.environ.get("LLDB_TEST_THREADS")
if num_threads_str:
num_threads = int(num_threads_str)
@@ -511,4 +496,8 @@ Run lldb test suite using a separate pro
sys.exit(exit_code)
if __name__ == '__main__':
- main()
+ sys.stderr.write(
+ "error: dosep.py no longer supports being called directly. "
+ "Please call dotest.py directly. The dosep.py-specific arguments "
+ "have been added under the Parallel processing arguments.\n")
+ sys.exit(128)
Modified: lldb/trunk/test/dotest.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/dotest.py?rev=246794&r1=246793&r2=246794&view=diff
==============================================================================
--- lldb/trunk/test/dotest.py (original)
+++ lldb/trunk/test/dotest.py Thu Sep 3 13:58:44 2015
@@ -29,8 +29,6 @@ import progress
import signal
import subprocess
import sys
-import textwrap
-import time
import inspect
import unittest2
import lldbtest_config
@@ -245,6 +243,13 @@ lldb_platform_name = None
lldb_platform_url = None
lldb_platform_working_dir = None
+# Parallel execution settings
+is_inferior_test_runner = False
+multiprocess_test_subdir = None
+num_threads = None
+output_on_success = False
+no_multiprocess_test_runner = False
+
def usage(parser):
parser.print_help()
if verbose > 0:
@@ -485,6 +490,11 @@ def parseOptionsAndInitTestdirs():
global lldb_platform_url
global lldb_platform_working_dir
global setCrashInfoHook
+ global is_inferior_test_runner
+ global multiprocess_test_subdir
+ global num_threads
+ global output_on_success
+ global no_multiprocess_test_runner
do_help = False
@@ -493,7 +503,7 @@ def parseOptionsAndInitTestdirs():
parser = dotest_args.create_parser()
args = dotest_args.parse_args(parser, sys.argv[1:])
-
+
if args.unset_env_varnames:
for env_var in args.unset_env_varnames:
if env_var in os.environ:
@@ -606,7 +616,7 @@ def parseOptionsAndInitTestdirs():
if args.d:
sys.stdout.write("Suspending the process %d to wait for debugger to attach...\n" % os.getpid())
- sys.stdout.flush()
+ sys.stdout.flush()
os.kill(os.getpid(), signal.SIGSTOP)
if args.e:
@@ -740,6 +750,21 @@ def parseOptionsAndInitTestdirs():
if dont_do_lldbmi_test and just_do_lldbmi_test:
usage(parser)
+ if args.no_multiprocess:
+ no_multiprocess_test_runner = True
+
+ if args.inferior:
+ is_inferior_test_runner = True
+
+ if args.output_on_success:
+ output_on_success = True
+
+ if args.num_threads:
+ num_threads = args.num_threads
+
+ if args.test_subdir:
+ multiprocess_test_subdir = args.test_subdir
+
if args.lldb_platform_name:
lldb_platform_name = args.lldb_platform_name
if args.lldb_platform_url:
@@ -1228,6 +1253,14 @@ def exitTestSuite(exitCode = None):
if exitCode:
sys.exit(exitCode)
+
+def isMultiprocessTestRunner():
+ # We're not multiprocess when we're either explicitly
+ # the inferior (as specified by the multiprocess test
+ # runner) OR we've been told to skip using the multiprocess
+ # test runner
+ return not (is_inferior_test_runner or no_multiprocess_test_runner)
+
# On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
# does not exist before proceeding to running the test suite.
if sys.platform.startswith("darwin"):
@@ -1239,6 +1272,14 @@ if sys.platform.startswith("darwin"):
# then, we walk the directory trees and collect the tests into our test suite.
#
parseOptionsAndInitTestdirs()
+
+# If we are running as the multiprocess test runner, kick off the
+# multiprocess test runner here.
+if isMultiprocessTestRunner():
+ import dosep
+ dosep.main(output_on_success, num_threads, multiprocess_test_subdir)
+ raise "should never get here"
+
setupSysPath()
setupCrashInfoHook()
Modified: lldb/trunk/test/dotest_args.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/dotest_args.py?rev=246794&r1=246793&r2=246794&view=diff
==============================================================================
--- lldb/trunk/test/dotest_args.py (original)
+++ lldb/trunk/test/dotest_args.py Thu Sep 3 13:58:44 2015
@@ -107,6 +107,33 @@ def create_parser():
group.set_defaults(disable_crash_dialog=True)
group.set_defaults(hide_inferior_console=True)
+ group = parser.add_argument_group('Parallel execution options')
+ group.add_argument(
+ '--inferior',
+ action='store_true',
+ help=('specify this invocation is a multiprocess inferior, '
+ 'used internally'))
+ group.add_argument(
+ '--no-multiprocess',
+ action='store_true',
+ help='skip running the multiprocess test runner')
+ group.add_argument(
+ '--output-on-success',
+ action='store_true',
+ help=('print full output of the dotest.py inferior, '
+ 'even when all tests succeed'))
+ group.add_argument(
+ '--threads',
+ type=int,
+ dest='num_threads',
+ help=('The number of threads/processes to use when running tests '
+ 'separately, defaults to the number of CPU cores available'))
+ parser.add_argument(
+ '--test-subdir',
+ action='store',
+ help='Specify a test subdirectory to use relative to the test root dir'
+ )
+
# Remove the reference to our helper function
del X
Modified: lldb/trunk/www/test.html
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/www/test.html?rev=246794&r1=246793&r2=246794&view=diff
==============================================================================
--- lldb/trunk/www/test.html (original)
+++ lldb/trunk/www/test.html Thu Sep 3 13:58:44 2015
@@ -67,19 +67,30 @@
</code>
<p>
- Besides <code>dotest.py</code>, there is also <code>dosep.py</code>, which runs
- multiple instances of <code>dotest.py</code> in parallel, thereby greatly
- decreasing the time it takes to run the full testsuite. The number of concurrent
- tests is controlled by the <code>LLDB_TEST_THREADS</code> environment variable or
- the <code>--threads</code> command line parameter. The default value is the number
- of CPUs on your system. To pass additional options to <code>dotest.py</code>,
- specify those options as an <code>-o</code> argument to <code>dosep.py</code>. For
- example, the command
+ The dotest.py script runs tests in parallel by default.
+ To disable the parallel test running feature, use the
+ <code>--no-multiprocess</code> flag. The number of
+ concurrent tests is controlled by
+ the <code>LLDB_TEST_THREADS</code> environment variable
+ or the <code>--threads</code> command line parameter.
+ The default value is the number of CPU cores on your
+ system.
</p>
- <code>python dosep.py -o "--executable bin/lldb -C bin/clang"</code>
<p>
- will specify the lldb and clang executables to test for each dotest invocation.
- <code>ninja check-lldb</code> is wrapper around <code>dosep.py</code>.
+ The parallel test running feature will handle an
+ additional <code>--test-subdir SUBDIR</code> arg. When
+ specified, SUBDIR is relative to the root test directory
+ and will limit all parallel test running to that
+ sudirectory's tree of tests.
+ </p>
+ <p>
+ The parallel test runner will run all tests within a
+ given directory serially, but will run multiple
+ directories concurrently. Thus, as a test writer, we
+ provide serialized test run semantics within a
+ directory. Note child directories are considered
+ entirely separate, so two child directories could be
+ running in parallel with a parent directory.
</p>
<h3>Running the test-suite remotely</h3>
More information about the lldb-commits
mailing list