[Lldb-commits] [lldb] r238467 - Refactor test runner to print sub-test-case pass/fail rate.
Zachary Turner
zturner at google.com
Thu May 28 12:56:26 PDT 2015
Author: zturner
Date: Thu May 28 14:56:26 2015
New Revision: 238467
URL: http://llvm.org/viewvc/llvm-project?rev=238467&view=rev
Log:
Refactor test runner to print sub-test-case pass/fail rate.
Modified:
lldb/trunk/test/api/multithreaded/Makefile
lldb/trunk/test/dosep.py
lldb/trunk/test/unittest2/result.py
lldb/trunk/test/unittest2/runner.py
Modified: lldb/trunk/test/api/multithreaded/Makefile
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/api/multithreaded/Makefile?rev=238467&r1=238466&r2=238467&view=diff
==============================================================================
--- lldb/trunk/test/api/multithreaded/Makefile (original)
+++ lldb/trunk/test/api/multithreaded/Makefile Thu May 28 14:56:26 2015
@@ -6,4 +6,4 @@ CXX_SOURCES := main.cpp
include $(LEVEL)/Makefile.rules
clean::
- rm -rf *.o *.d *.dSYM
+ rm $(wildcard -rf *.o *.d *.dSYM)
Modified: lldb/trunk/test/dosep.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/dosep.py?rev=238467&r1=238466&r2=238467&view=diff
==============================================================================
--- lldb/trunk/test/dosep.py (original)
+++ lldb/trunk/test/dosep.py Thu May 28 14:56:26 2015
@@ -66,27 +66,47 @@ default_timeout = os.getenv("LLDB_TEST_T
# Status codes for running command with timeout.
eTimedOut, ePassed, eFailed = 124, 0, 1
+def parse_test_results(output):
+ passes = 0
+ failures = 0
+ for result in output:
+ pass_count = re.search("^RESULT:.*([0-9]+) passes", result, re.MULTILINE)
+ fail_count = re.search("^RESULT:.*([0-9]+) failures", result, re.MULTILINE)
+ error_count = re.search("^RESULT:.*([0-9]+) errors", result, re.MULTILINE)
+ this_fail_count = 0
+ this_error_count = 0
+ if pass_count != None:
+ passes = passes + int(pass_count.group(1))
+ if fail_count != None:
+ failures = failures + int(fail_count.group(1))
+ if error_count != None:
+ failures = failures + int(error_count.group(1))
+ pass
+ return passes, failures
+
def call_with_timeout(command, timeout):
"""Run command with a timeout if possible."""
"""-s QUIT will create a coredump if they are enabled on your system"""
+ process = None
+ if timeout_command and timeout != "0":
+ command = [timeout_command, '-s', 'QUIT', timeout] + command
+ # Specifying a value for close_fds is unsupported on Windows when using subprocess.PIPE
if os.name != "nt":
- if timeout_command and timeout != "0":
- return subprocess.call([timeout_command, '-s', 'QUIT', timeout] + command,
- stdin=subprocess.PIPE, close_fds=True)
- return (ePassed if subprocess.call(command, stdin=subprocess.PIPE, close_fds=True) == 0
- else eFailed)
+ process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
else:
- if timeout_command and timeout != "0":
- return subprocess.call([timeout_command, '-s', 'QUIT', timeout] + command,
- stdin=subprocess.PIPE)
- return (ePassed if subprocess.call(command, stdin=subprocess.PIPE) == 0
- else eFailed)
+ process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ output = process.communicate()
+ exit_status = process.returncode
+ passes, failures = parse_test_results(output)
+ return exit_status, passes, failures
def process_dir(root, files, test_root, dotest_argv):
"""Examine a directory for tests, and invoke any found within it."""
timed_out = []
failed = []
passed = []
+ pass_sub_count = 0
+ fail_sub_count = 0
for name in files:
path = os.path.join(root, name)
@@ -107,15 +127,18 @@ def process_dir(root, files, test_root,
timeout = os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or default_timeout
- exit_status = call_with_timeout(command, timeout)
+ exit_status, pass_count, fail_count = call_with_timeout(command, timeout)
+
+ pass_sub_count = pass_sub_count + pass_count
+ fail_sub_count = fail_sub_count + fail_count
- if ePassed == exit_status:
+ if exit_status == ePassed:
passed.append(name)
else:
if eTimedOut == exit_status:
timed_out.append(name)
failed.append(name)
- return (timed_out, failed, passed)
+ return (timed_out, failed, passed, fail_sub_count, pass_sub_count)
in_q = None
out_q = None
@@ -154,14 +177,18 @@ def walk_and_invoke(test_directory, test
timed_out = []
failed = []
passed = []
+ fail_sub_count = 0
+ pass_sub_count = 0
for test_result in test_results:
- (dir_timed_out, dir_failed, dir_passed) = test_result
+ (dir_timed_out, dir_failed, dir_passed, dir_fail_sub_count, dir_pass_sub_count) = test_result
timed_out += dir_timed_out
failed += dir_failed
passed += dir_passed
+ fail_sub_count = fail_sub_count + dir_fail_sub_count
+ pass_sub_count = pass_sub_count + dir_pass_sub_count
- return (timed_out, failed, passed)
+ return (timed_out, failed, passed, fail_sub_count, pass_sub_count)
def getExpectedTimeouts(platform_name):
# returns a set of test filenames that might timeout
@@ -294,10 +321,11 @@ Run lldb test suite using a separate pro
num_threads = 1
system_info = " ".join(platform.uname())
- (timed_out, failed, passed) = walk_and_invoke(test_directory, test_subdir, dotest_argv,
- num_threads)
+ (timed_out, failed, passed, all_fails, all_passes) = walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads)
+
timed_out = set(timed_out)
- num_tests = len(failed) + len(passed)
+ num_test_files = len(failed) + len(passed)
+ num_tests = all_fails + all_passes
# move core files into session dir
cores = find('core.*', test_subdir)
@@ -322,10 +350,10 @@ Run lldb test suite using a separate pro
test_name = os.path.splitext(xtime)[0]
touch(os.path.join(session_dir, "{}-{}".format(result, test_name)))
- print "Ran %d tests." % num_tests
+ print "Ran %d test suites (%d failed) (%f%%)" % (num_test_files, len(failed), 100.0*len(failed)/num_test_files)
+ print "Ran %d test cases (%d failed) (%f%%)" % (num_tests, all_fails, 100.0*all_fails/num_tests)
if len(failed) > 0:
failed.sort()
- print "Failing Tests (%d)" % len(failed)
for f in failed:
print "%s: LLDB (suite) :: %s (%s)" % (
"TIMEOUT" if f in timed_out else "FAIL", f, system_info
Modified: lldb/trunk/test/unittest2/result.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/unittest2/result.py?rev=238467&r1=238466&r2=238467&view=diff
==============================================================================
--- lldb/trunk/test/unittest2/result.py (original)
+++ lldb/trunk/test/unittest2/result.py Thu May 28 14:56:26 2015
@@ -40,6 +40,7 @@ class TestResult(unittest.TestResult):
def __init__(self):
self.failfast = False
self.failures = []
+ self.passes = []
self.errors = []
self.testsRun = 0
self.skipped = []
@@ -117,6 +118,7 @@ class TestResult(unittest.TestResult):
def addSuccess(self, test):
"Called when a test has completed successfully"
+ self.passes.append(test)
pass
def addSkip(self, test, reason):
Modified: lldb/trunk/test/unittest2/runner.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/unittest2/runner.py?rev=238467&r1=238466&r2=238467&view=diff
==============================================================================
--- lldb/trunk/test/unittest2/runner.py (original)
+++ lldb/trunk/test/unittest2/runner.py Thu May 28 14:56:26 2015
@@ -175,32 +175,29 @@ class TextTestRunner(unittest.TextTestRu
(run, run != 1 and "s" or "", timeTaken))
self.stream.writeln()
- expectedFails = unexpectedSuccesses = skipped = 0
+ expectedFails = unexpectedSuccesses = skipped = passed = failed = errored = 0
try:
results = map(len, (result.expectedFailures,
result.unexpectedSuccesses,
- result.skipped))
- expectedFails, unexpectedSuccesses, skipped = results
+ result.skipped,
+ result.passes,
+ result.failures,
+ result.errors))
+ expectedFails, unexpectedSuccesses, skipped, passed, failed, errored = results
except AttributeError:
pass
infos = []
+ infos.append("%d passes" % passed)
+ infos.append("%d failures" % failed)
+ infos.append("%d errors" % errored)
+ infos.append("%d skipped" % skipped)
+ infos.append("%d expected failures" % expectedFails)
+ infos.append("%d unexpected successes" % unexpectedSuccesses)
+ self.stream.write("RESULT: ")
if not result.wasSuccessful():
self.stream.write("FAILED")
- failed, errored = map(len, (result.failures, result.errors))
- if failed:
- infos.append("failures=%d" % failed)
- if errored:
- infos.append("errors=%d" % errored)
else:
- self.stream.write("OK")
- if skipped:
- infos.append("skipped=%d" % skipped)
- if expectedFails:
- infos.append("expected failures=%d" % expectedFails)
- if unexpectedSuccesses:
- infos.append("unexpected successes=%d" % unexpectedSuccesses)
- if infos:
- self.stream.writeln(" (%s)" % (", ".join(infos),))
- else:
- self.stream.write("\n")
+ self.stream.write("PASSED")
+
+ self.stream.writeln(" (%s)" % (", ".join(infos),))
return result
More information about the lldb-commits
mailing list