[Lldb-commits] [lldb] r175309 - Improve test harness for the buildbots
Daniel Malea
daniel.malea at intel.com
Fri Feb 15 13:31:37 PST 2013
Author: dmalea
Date: Fri Feb 15 15:31:37 2013
New Revision: 175309
URL: http://llvm.org/viewvc/llvm-project?rev=175309&view=rev
Log:
Improve test harness for the buildbots
- Add a "parsable" mode to dotest.py that outputs test results in exactly the same format as clang's lit tests
- Improve dosep script to output list of failing tests (output should look like clang test failure summaries)
- Cleanup lldb/test/Makefile to remove needless parameters and environment variables
- Switch makefile tests to use parsable-mode output; should make the buildbot results parsable
- Switch makefile tests to use dosep to log catch crashing tests (instead of halting the test suite)
Modified:
lldb/trunk/test/Makefile
lldb/trunk/test/dosep.ty
lldb/trunk/test/dotest.py
Modified: lldb/trunk/test/Makefile
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/Makefile?rev=175309&r1=175308&r2=175309&view=diff
==============================================================================
--- lldb/trunk/test/Makefile (original)
+++ lldb/trunk/test/Makefile Fri Feb 15 15:31:37 2013
@@ -29,10 +29,5 @@ clean::
# Run the tests
#----------------------------------------------------------------------
check-local::
- rm -rf ../test-rdir
- env PATH="$(ToolDir):$(PATH)" \
- PYTHONPATH="$(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib(True, False, \"$(LibDir)/..\")" )" \
- LLDB_EXEC=$(ToolDir)/lldb \
- LLDB_BUILD_TYPE=Makefile \
- $(SHLIBPATH_VAR)=$(LibDir):$($(SHLIBPATH_VAR)) \
- python $(PROJ_SRC_DIR)/dotest.py -i -v -r ../test-rdir
+ rm -rf lldb-test-traces
+ python $(PROJ_SRC_DIR)/dosep.ty -o "--executable $(ToolDir)/lldb -q -s lldb-test-traces"
Modified: lldb/trunk/test/dosep.ty
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/dosep.ty?rev=175309&r1=175308&r2=175309&view=diff
==============================================================================
--- lldb/trunk/test/dosep.ty (original)
+++ lldb/trunk/test/dosep.ty Fri Feb 15 15:31:37 2013
@@ -12,6 +12,8 @@ template = '%s/dotest.py %s -p %s %s'
def walk_and_invoke(test_root, dotest_options):
"""Look for matched file and invoke test driver on it."""
+ failed = []
+ passed = []
for root, dirs, files in os.walk(test_root, topdown=False):
for name in files:
path = os.path.join(root, name)
@@ -25,8 +27,13 @@ def walk_and_invoke(test_root, dotest_op
continue
command = template % (test_root, dotest_options if dotest_options else "", name, root)
- print "Running %s" % (command)
- os.system(command)
+ if "-v" in dotest_options:
+ print "Running %s" % (command)
+ if 0 != os.system(command):
+ failed.append(name)
+ else:
+ passed.append(name)
+ return (failed, passed)
def main():
test_root = sys.path[0]
@@ -44,8 +51,15 @@ Run lldb test suite using a separate pro
print "dotest.py options:", dotest_options
- walk_and_invoke(test_root, dotest_options)
-
+ (failed, passed) = walk_and_invoke(test_root, dotest_options)
+ num_tests = len(failed) + len(passed)
+ print "Ran %d tests." % num_tests
+ if len(failed) > 0:
+ print "Failing Tests (%d)" % len(failed)
+ for f in failed:
+ print "\tLLDB :: %s" % f
+ sys.exit(1)
+ sys.exit(0)
if __name__ == '__main__':
main()
Modified: lldb/trunk/test/dotest.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/dotest.py?rev=175309&r1=175308&r2=175309&view=diff
==============================================================================
--- lldb/trunk/test/dotest.py (original)
+++ lldb/trunk/test/dotest.py Fri Feb 15 15:31:37 2013
@@ -187,6 +187,10 @@ skip_long_running_test = True
# turn it off.
noHeaders = False
+# Parsable mode silences headers, and any other output this script might generate, and instead
+# prints machine-readable output similar to what clang tests produce.
+parsable = False
+
# The regular expression pattern to match against eligible filenames as our test cases.
regexp = None
@@ -341,6 +345,7 @@ def parseOptionsAndInitTestdirs():
global skip_build_and_cleanup
global skip_long_running_test
global noHeaders
+ global parsable
global regexp
global rdir
global sdir_name
@@ -401,6 +406,7 @@ def parseOptionsAndInitTestdirs():
X('-F', 'Fail fast. Stop the test suite on the first error/failure')
X('-i', "Ignore (don't bailout) if 'lldb.py' module cannot be located in the build tree relative to this script; use PYTHONPATH to locate the module")
X('-n', "Don't print the headers like build dir, lldb version, and svn info at all")
+ X('-q', "Don't print extra output from this script.")
X('-S', "Skip the build and cleanup while running the test. Use this option with care as you would need to build the inferior(s) by hand and build the executable(s) with the correct name(s). This can be used with '-# n' to stress test certain test cases for n number of times")
X('-t', 'Turn on tracing of lldb command and other detailed test executions')
group.add_argument('-u', dest='unset_env_varnames', metavar='variable', action='append', help='Specify an environment variable to unset before running the test cases. e.g., -u DYLD_INSERT_LIBRARIES -u MallocScribble')
@@ -425,8 +431,8 @@ def parseOptionsAndInitTestdirs():
del os.environ[env_var]
#os.unsetenv(env_var)
- # only print the args if being verbose
- if args.v:
+ # only print the args if being verbose (and parsable is off)
+ if args.v and not args.q:
print args
if args.h:
@@ -545,6 +551,10 @@ def parseOptionsAndInitTestdirs():
usage(parser)
regexp = args.p
+ if args.q:
+ noHeaders = True
+ parsable = True
+
if args.R:
if args.R.startswith('-'):
usage(parser)
@@ -1111,8 +1121,9 @@ def getsource_if_available(obj):
except:
return repr(obj)
-print "lldb.pre_flight:", getsource_if_available(lldb.pre_flight)
-print "lldb.post_flight:", getsource_if_available(lldb.post_flight)
+if not noHeaders:
+ print "lldb.pre_flight:", getsource_if_available(lldb.pre_flight)
+ print "lldb.post_flight:", getsource_if_available(lldb.post_flight)
# Put all these test decorators in the lldb namespace.
lldb.dont_do_python_api_test = dont_do_python_api_test
@@ -1200,7 +1211,8 @@ for i in range(len(compilers)):
compilers[i] = cmd_output.split('\n')[0]
print "'xcrun -find %s' returning %s" % (c, compilers[i])
-print "compilers=%s" % str(compilers)
+if not parsable:
+ print "compilers=%s" % str(compilers)
if not compilers or len(compilers) == 0:
print "No eligible compiler found, exiting."
@@ -1280,16 +1292,18 @@ for ia in range(len(archs) if iterArchs
sys.path = [x.replace(rdir, newrdir, 1) for x in old_sys_path]
# Output the configuration.
- sys.stderr.write("\nConfiguration: " + configString + "\n")
+ if not parsable:
+ sys.stderr.write("\nConfiguration: " + configString + "\n")
#print "sys.stderr name is", sys.stderr.name
#print "sys.stdout name is", sys.stdout.name
# First, write out the number of collected test cases.
- sys.stderr.write(separator + "\n")
- sys.stderr.write("Collected %d test%s\n\n"
- % (suite.countTestCases(),
- suite.countTestCases() != 1 and "s" or ""))
+ if not parsable:
+ sys.stderr.write(separator + "\n")
+ sys.stderr.write("Collected %d test%s\n\n"
+ % (suite.countTestCases(),
+ suite.countTestCases() != 1 and "s" or ""))
class LLDBTestResult(unittest2.TextTestResult):
"""
@@ -1343,12 +1357,17 @@ for ia in range(len(archs) if iterArchs
self.counter = 0
(width, height) = LLDBTestResult.getTerminalSize()
self.progressbar = None
- if width > 10:
+ if width > 10 and not parsable and progress_bar:
try:
self.progressbar = progress.ProgressWithEvents(stdout=self.stream,start=0,end=suite.countTestCases(),width=width-10)
except:
self.progressbar = None
+ def _config_string(self, test):
+ compiler = getattr(test, "getCompiler", None)
+ arch = getattr(test, "getArchitecture", None)
+ return "%s-%s" % (compiler() if compiler else "", arch() if arch else "")
+
def _exc_info_to_string(self, err, test):
"""Overrides superclass TestResult's method in order to append
our test config info string to the exception info string."""
@@ -1397,22 +1416,34 @@ for ia in range(len(archs) if iterArchs
self.stream.write(self.fmt % self.counter)
super(LLDBTestResult, self).startTest(test)
+ def addSuccess(self, test):
+ global parsable
+ super(LLDBTestResult, self).addSuccess(test)
+ if parsable:
+ self.stream.write("PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
+
def addError(self, test, err):
global sdir_has_content
+ global parsable
sdir_has_content = True
super(LLDBTestResult, self).addError(test, err)
method = getattr(test, "markError", None)
if method:
method()
+ if parsable:
+ self.stream.write("ERROR: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
def addFailure(self, test, err):
global sdir_has_content
global failuresPerCategory
+ global parsable
sdir_has_content = True
super(LLDBTestResult, self).addFailure(test, err)
method = getattr(test, "markFailure", None)
if method:
method()
+ if parsable:
+ self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
if useCategories:
test_categories = self.getCategoriesForTest(test)
for category in test_categories:
@@ -1423,32 +1454,48 @@ for ia in range(len(archs) if iterArchs
def addExpectedFailure(self, test, err):
global sdir_has_content
+ global parsable
sdir_has_content = True
super(LLDBTestResult, self).addExpectedFailure(test, err)
method = getattr(test, "markExpectedFailure", None)
if method:
method()
+ if parsable:
+ self.stream.write("XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
def addSkip(self, test, reason):
global sdir_has_content
+ global parsable
sdir_has_content = True
super(LLDBTestResult, self).addSkip(test, reason)
method = getattr(test, "markSkippedTest", None)
if method:
method()
+ if parsable:
+ self.stream.write("UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % (self._config_string(test), str(test), reason))
def addUnexpectedSuccess(self, test):
global sdir_has_content
+ global parsable
sdir_has_content = True
super(LLDBTestResult, self).addUnexpectedSuccess(test)
method = getattr(test, "markUnexpectedSuccess", None)
if method:
method()
+ if parsable:
+ self.stream.write("XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
+
+ if parsable:
+ v = 0
+ elif progress_bar:
+ v = 1
+ else:
+ v = verbose
# Invoke the test runner.
if count == 1:
result = unittest2.TextTestRunner(stream=sys.stderr,
- verbosity=(1 if progress_bar else verbose),
+ verbosity=v,
failfast=failfast,
resultclass=LLDBTestResult).run(suite)
else:
@@ -1457,13 +1504,14 @@ for ia in range(len(archs) if iterArchs
# not enforced.
LLDBTestResult.__ignore_singleton__ = True
for i in range(count):
+
result = unittest2.TextTestRunner(stream=sys.stderr,
- verbosity=(1 if progress_bar else verbose),
+ verbosity=v,
failfast=failfast,
resultclass=LLDBTestResult).run(suite)
-if sdir_has_content:
+if sdir_has_content and not parsable:
sys.stderr.write("Session logs for test failures/errors/unexpected successes"
" can be found in directory '%s'\n" % sdir_name)
More information about the lldb-commits
mailing list