[Lldb-commits] [lldb] r254530 - Adds candidate formatter for replacing legacy summary results.
Todd Fiala via lldb-commits
lldb-commits at lists.llvm.org
Wed Dec 2 10:48:38 PST 2015
Author: tfiala
Date: Wed Dec 2 12:48:38 2015
New Revision: 254530
URL: http://llvm.org/viewvc/llvm-project?rev=254530&view=rev
Log:
Adds candidate formatter for replacing legacy summary results.
Also cleans up some usages of strings where symbolic names
were safer and made more sense.
Try a test run with something like this to check out the new
basic results formatter (not used by default):
time test/dotest.py --executable `pwd`/build/Debug/lldb --results-formatter lldbsuite.test.basic_results_formatter.BasicResultsFormatter --results-file stdout
This will yield something like:
Testing: 1 test suites, 8 threads
1 out of 1 test suites processed - TestHelp.py
Test Results
Total Test Methods Run (excluding reruns): 13
Test Method rerun count: 0
===================
Test Result Summary
===================
Success: 13
Expected Failure: 0
Failure: 0
Error: 0
Unexpected Success: 0
Skip: 0
Whereas something with a bit of error will look more like this:
42 out of 42 test suites processed - TestSymbolTable.py
Test Results
Total Test Methods Run (excluding reruns): 166
Test Method rerun count: 0
===================
Test Result Summary
===================
Success: 93
Expected Failure: 10
Failure: 2
Error: 2
Unexpected Success: 0
Skip: 59
Details:
FAIL:
TestModulesInlineFunctions.ModulesInlineFunctionsTestCase.test_expr_dsym
(/Users/tfiala/work/lldb-tot/git-svn/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/TestModulesInlineFunctions.py)
FAIL:
TestModulesInlineFunctions.ModulesInlineFunctionsTestCase.test_expr_dwarf
(/Users/tfiala/work/lldb-tot/git-svn/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/TestModulesInlineFunctions.py)
ERROR: TestObjCCheckers.ObjCCheckerTestCase.test_objc_checker_dsym
(/Users/tfiala/work/lldb-tot/git-svn/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/TestObjCCheckers.py)
ERROR: TestObjCCheckers.ObjCCheckerTestCase.test_objc_checker_dwarf
(/Users/tfiala/work/lldb-tot/git-svn/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/TestObjCCheckers.py)
The Details header only prints if there are any issues to report. The
Details section has tags that should get picked up using the normal
issue text scrapers (e.g. buildbot).
Test numbers reported are strictly test method runs.
The rerun bit at the top is in support of the multi-pass test
runner code (to run the low-load, single worker test pass for
tests that failed the first run), which I'll be able to put up
for review after this.
ResultsFormatters now have the ability to indicate they replace
the legacy summary, as this one does.
Once we come to agreement on the exact format, I will switch
us over to using this by default.
Added:
lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py
Modified:
lldb/trunk/packages/Python/lldbsuite/test/curses_results.py
lldb/trunk/packages/Python/lldbsuite/test/dosep.py
lldb/trunk/packages/Python/lldbsuite/test/test_results.py
Added: lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py?rev=254530&view=auto
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py (added)
+++ lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py Wed Dec 2 12:48:38 2015
@@ -0,0 +1,259 @@
+"""
+ The LLVM Compiler Infrastructure
+
+This file is distributed under the University of Illinois Open Source
+License. See LICENSE.TXT for details.
+
+Provides basic test result output. This is intended to be suitable for
+normal LLDB test run output when no other option is specified.
+"""
+from __future__ import print_function
+
+from . import test_results
+
+
+class BasicResultsFormatter(test_results.ResultsFormatter):
+ """Provides basic test result output."""
+ @classmethod
+ def arg_parser(cls):
+ """@return arg parser used to parse formatter-specific options."""
+ parser = super(BasicResultsFormatter, cls).arg_parser()
+
+ parser.add_argument(
+ "--assert-on-unknown-events",
+ action="store_true",
+ help=('cause unknown test events to generate '
+ 'a python assert. Default is to ignore.'))
+ return parser
+
+ def __init__(self, out_file, options):
+ """Initializes the BasicResultsFormatter instance.
+ @param out_file file-like object where formatted output is written.
+ @param options_dict specifies a dictionary of options for the
+ formatter.
+ """
+ # Initialize the parent
+ super(BasicResultsFormatter, self).__init__(out_file, options)
+
+ # self.result_event will store the most current result_event
+ # by test method
+ self.result_events = {}
+ self.test_method_rerun_count = 0
+
+ def handle_event(self, test_event):
+ super(BasicResultsFormatter, self).handle_event(test_event)
+ if test_event is None:
+ return
+
+ event_type = test_event["event"]
+ if event_type is None:
+ return
+
+ if event_type == "terminate":
+ self._finish_output()
+ elif event_type == "test_start":
+ self.track_start_time(
+ test_event["test_class"],
+ test_event["test_name"],
+ test_event["event_time"])
+ elif event_type == "test_result":
+ # Build the test key.
+ test_key = test_event.get("test_filename", None)
+ if test_key is None:
+ raise Exception(
+ "failed to find test filename for test event {}".format(
+ test_event))
+ test_key += ".{}.{}".format(
+ test_event.get("test_class", ""),
+ test_event.get("test_name", ""))
+
+ # Save the most recent test event for the test key.
+ # This allows a second test phase to overwrite the most
+ # recent result for the test key (unique per method).
+ # We do final reporting at the end, so we'll report based
+ # on final results.
+ # We do this so that a re-run caused by, perhaps, the need
+ # to run a low-load, single-worker test run can have the final
+ # run's results to always be used.
+ if test_key in self.result_events:
+ # We are replacing the result of something that was
+ # already counted by the base class. Remove the double
+ # counting by reducing by one the count for the test
+ # result status.
+ old_status = self.result_events[test_key]["status"]
+ self.result_status_counts[old_status] -= 1
+
+ self.test_method_rerun_count += 1
+ if self.options.warn_on_multiple_results:
+ print(
+ "WARNING: test key {} already has a result: "
+ "old:{} new:{}",
+ self.result_events[test_key],
+ test_event)
+ self.result_events[test_key] = test_event
+ else:
+ # This is an unknown event.
+ if self.options.assert_on_unknown_events:
+ raise Exception("unknown event type {} from {}\n".format(
+ event_type, test_event))
+
+ def _partition_results_by_status(self, categories):
+ """Partitions the captured test results by event status.
+
+ This permits processing test results by the category ids.
+
+ @param categories the list of categories on which to partition.
+ Follows the format described in _report_category_details().
+
+ @return a dictionary where each key is the test result status,
+ and each entry is a list containing all the test result events
+ that matched that test result status. Result status IDs with
+ no matching entries will have a zero-length list.
+ """
+ partitioned_events = {}
+ for category in categories:
+ result_status_id = category[0]
+ matching_events = [
+ [key, event] for (key, event) in self.result_events.items()
+ if event.get("status", "") == result_status_id]
+ partitioned_events[result_status_id] = sorted(
+ matching_events,
+ key=lambda x: x[1]["test_name"])
+ return partitioned_events
+
+ def _print_summary_counts(self, categories, result_events_by_status):
+ """Prints summary counts for all categories.
+
+ @param categories the list of categories on which to partition.
+ Follows the format described in _report_category_details().
+
+ @param result_events_by_status the partitioned list of test
+ result events in a dictionary, with the key set to the test
+ result status id and the value set to the list of test method
+ results that match the status id.
+ """
+
+ # Get max length for category printed name
+ category_with_max_printed_name = max(
+ categories, key=lambda x: len(x[1]))
+ max_category_name_length = len(category_with_max_printed_name[1])
+
+ banner_text = "Test Result Summary"
+ banner_separator = "".ljust(len(banner_text), "=")
+
+ self.out_file.write("\n{}\n{}\n{}\n".format(
+ banner_separator,
+ banner_text,
+ banner_separator))
+
+ for category in categories:
+ result_status_id = category[0]
+ result_label = "{}:".format(category[1]).ljust(
+ max_category_name_length + 1)
+ count = len(result_events_by_status[result_status_id])
+ self.out_file.write("{} {:4}\n".format(
+ result_label,
+ count))
+
+ @classmethod
+ def _has_printable_details(cls, categories, result_events_by_status):
+ """Returns whether there are any test result details that need to be printed.
+
+ This will spin through the results and see if any result in a category
+ that is printable has any results to print.
+
+ @param categories the list of categories on which to partition.
+ Follows the format described in _report_category_details().
+
+ @param result_events_by_status the partitioned list of test
+ result events in a dictionary, with the key set to the test
+ result status id and the value set to the list of test method
+ results that match the status id.
+
+ @return True if there are any details (i.e. test results
+ for failures, errors, unexpected successes); False otherwise.
+ """
+ for category in categories:
+ result_status_id = category[0]
+ print_matching_tests = category[2]
+ if print_matching_tests:
+ if len(result_events_by_status[result_status_id]) > 0:
+ # We found a printable details test result status
+ # that has details to print.
+ return True
+ # We didn't find any test result category with printable
+ # details.
+ return False
+
+ def _report_category_details(self, category, result_events_by_status):
+ """Reports all test results matching the given category spec.
+
+ @param category a category spec of the format [test_event_name,
+ printed_category_name, print_matching_entries?]
+
+ @param result_events_by_status the partitioned list of test
+ result events in a dictionary, with the key set to the test
+ result status id and the value set to the list of test method
+ results that match the status id.
+ """
+ result_status_id = category[0]
+ print_matching_tests = category[2]
+ detail_label = category[3]
+
+ if print_matching_tests:
+ # Sort by test name
+ for (_, event) in result_events_by_status[result_status_id]:
+ self.out_file.write("{}: {}.{} ({})\n".format(
+ detail_label,
+ event["test_class"],
+ event["test_name"],
+ event["test_filename"]))
+
+ def _finish_output_no_lock(self):
+ """Writes the test result report to the output file."""
+ self.out_file.write("\nTest Results\n")
+ self.out_file.write(
+ "Total Test Methods Run (excluding reruns): {}\n".format(
+ len(self.result_events)))
+ self.out_file.write("Test Method rerun count: {}\n".format(
+ self.test_method_rerun_count))
+
+ # Output each of the test result entries.
+ categories = [
+ # result id, printed name, print matching tests?, detail label
+ [test_results.EventBuilder.STATUS_SUCCESS,
+ "Success", False, None],
+ [test_results.EventBuilder.STATUS_EXPECTED_FAILURE,
+ "Expected Failure", False, None],
+ [test_results.EventBuilder.STATUS_FAILURE,
+ "Failure", True, "FAIL"],
+ [test_results.EventBuilder.STATUS_ERROR, "Error", True, "ERROR"],
+ [test_results.EventBuilder.STATUS_UNEXPECTED_SUCCESS,
+ "Unexpected Success", True, "UNEXPECTED SUCCESS"],
+ [test_results.EventBuilder.STATUS_SKIP, "Skip", False, None]]
+
+ # Partition all the events by test result status
+ result_events_by_status = self._partition_results_by_status(
+ categories)
+
+ # Print the summary
+ self._print_summary_counts(categories, result_events_by_status)
+
+ # Print the details
+ have_details = self._has_printable_details(
+ categories, result_events_by_status)
+ if have_details:
+ self.out_file.write("\nDetails:\n")
+ for category in categories:
+ self._report_category_details(
+ category, result_events_by_status)
+
+ def _finish_output(self):
+ """Prepare and write the results report as all incoming events have
+ arrived.
+ """
+ with self.lock:
+ self._finish_output_no_lock()
+
+ def replaces_summary(self):
+ return True
Modified: lldb/trunk/packages/Python/lldbsuite/test/curses_results.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/curses_results.py?rev=254530&r1=254529&r2=254530&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/curses_results.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/curses_results.py Wed Dec 2 12:48:38 2015
@@ -24,6 +24,8 @@ import time
# LLDB modules
from . import lldbcurses
from . import test_results
+from .test_results import EventBuilder
+
class Curses(test_results.ResultsFormatter):
"""Receives live results from tests that are running and reports them to the terminal in a curses GUI"""
@@ -38,7 +40,7 @@ class Curses(test_results.ResultsFormatt
self.job_tests = [None] * 64
self.results = list()
try:
- self.main_window = lldbcurses.intialize_curses()
+ self.main_window = lldbcurses.intialize_curses()
self.main_window.add_key_action('\t', self.main_window.select_next_first_responder, "Switch between views that can respond to keyboard input")
self.main_window.refresh()
self.job_panel = None
@@ -53,26 +55,25 @@ class Curses(test_results.ResultsFormatt
self.using_terminal = False
print("Unexpected error:", sys.exc_info()[0])
raise
-
-
+
self.line_dict = dict()
- #self.events_file = open("/tmp/events.txt", "w")
+ # self.events_file = open("/tmp/events.txt", "w")
# self.formatters = list()
# if tee_results_formatter:
# self.formatters.append(tee_results_formatter)
def status_to_short_str(self, status):
- if status == 'success':
+ if status == EventBuilder.STATUS_SUCCESS:
return '.'
- elif status == 'failure':
+ elif status == EventBuilder.STATUS_FAILURE:
return 'F'
- elif status == 'unexpected_success':
+ elif status == EventBuilder.STATUS_UNEXPECTED_SUCCESS:
return '?'
- elif status == 'expected_failure':
+ elif status == EventBuilder.STATUS_EXPECTED_FAILURE:
return 'X'
- elif status == 'skip':
+ elif status == EventBuilder.STATUS_SKIP:
return 'S'
- elif status == 'error':
+ elif status == EventBuilder.STATUS_ERROR:
return 'E'
else:
return status
@@ -88,7 +89,7 @@ class Curses(test_results.ResultsFormatt
self.info_panel.top()
else:
self.info_panel.show()
-
+
self.main_window.push_first_responder(self.info_panel)
test_start = self.results[selected_idx][0]
test_result = self.results[selected_idx][1]
@@ -99,9 +100,9 @@ class Curses(test_results.ResultsFormatt
def hide_info_panel(self):
self.main_window.pop_first_responder(self.info_panel)
- self.info_panel.hide()
+ self.info_panel.hide()
self.main_window.refresh()
-
+
def toggle_status(self, status):
if status:
# Toggle showing and hiding results whose status matches "status" in "Results" window
@@ -123,8 +124,8 @@ class Curses(test_results.ResultsFormatt
name = test_result['test_class'] + '.' + test_result['test_name']
self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status), test_result['elapsed_time'], name))
if update:
- self.main_window.refresh()
-
+ self.main_window.refresh()
+
def handle_event(self, test_event):
with self.lock:
super(Curses, self).handle_event(test_event)
@@ -137,7 +138,7 @@ class Curses(test_results.ResultsFormatt
if 'event' in test_event:
check_for_one_key = True
#print(str(test_event), file=self.events_file)
- event = test_event['event']
+ event = test_event['event']
if self.status_panel:
self.status_panel.update_status('time', str(datetime.timedelta(seconds=math.floor(time.time() - self.start_time))))
if event == 'test_start':
@@ -176,7 +177,7 @@ class Curses(test_results.ResultsFormatt
elif event == 'job_end':
self.jobs[worker_index] = ''
self.job_panel.set_line(worker_index, '')
- elif event == 'initialize':
+ elif event == 'initialize':
self.initialize_event = test_event
num_jobs = test_event['worker_count']
job_frame = self.main_window.get_contained_rect(height=num_jobs+2)
@@ -184,41 +185,40 @@ class Curses(test_results.ResultsFormatt
status_frame = self.main_window.get_contained_rect(height=1, top_inset=self.main_window.get_size().h-1)
self.job_panel = lldbcurses.BoxedPanel(frame=job_frame, title="Jobs")
self.results_panel = lldbcurses.BoxedPanel(frame=results_frame, title="Results")
-
+
self.results_panel.add_key_action(curses.KEY_UP, self.results_panel.select_prev , "Select the previous list entry")
self.results_panel.add_key_action(curses.KEY_DOWN, self.results_panel.select_next , "Select the next list entry")
self.results_panel.add_key_action(curses.KEY_HOME, self.results_panel.scroll_begin , "Scroll to the start of the list")
self.results_panel.add_key_action(curses.KEY_END, self.results_panel.scroll_end , "Scroll to the end of the list")
self.results_panel.add_key_action(curses.KEY_ENTER, self.show_info_panel , "Display info for the selected result item")
- self.results_panel.add_key_action('.', lambda : self.toggle_status('success') , "Toggle showing/hiding tests whose status is 'success'")
- self.results_panel.add_key_action('e', lambda : self.toggle_status('error') , "Toggle showing/hiding tests whose status is 'error'")
- self.results_panel.add_key_action('f', lambda : self.toggle_status('failure') , "Toggle showing/hiding tests whose status is 'failure'")
- self.results_panel.add_key_action('s', lambda : self.toggle_status('skip') , "Toggle showing/hiding tests whose status is 'skip'")
- self.results_panel.add_key_action('x', lambda : self.toggle_status('expected_failure') , "Toggle showing/hiding tests whose status is 'expected_failure'")
- self.results_panel.add_key_action('?', lambda : self.toggle_status('unexpected_success'), "Toggle showing/hiding tests whose status is 'unexpected_success'")
+ self.results_panel.add_key_action('.', lambda : self.toggle_status(EventBuilder.STATUS_SUCCESS) , "Toggle showing/hiding tests whose status is 'success'")
+ self.results_panel.add_key_action('e', lambda : self.toggle_status(EventBuilder.STATUS_ERROR) , "Toggle showing/hiding tests whose status is 'error'")
+ self.results_panel.add_key_action('f', lambda : self.toggle_status(EventBuilder.STATUS_FAILURE) , "Toggle showing/hiding tests whose status is 'failure'")
+ self.results_panel.add_key_action('s', lambda : self.toggle_status(EventBuilder.STATUS_SKIP) , "Toggle showing/hiding tests whose status is 'skip'")
+ self.results_panel.add_key_action('x', lambda : self.toggle_status(EventBuilder.STATUS_EXPECTED_FAILURE) , "Toggle showing/hiding tests whose status is 'expected_failure'")
+ self.results_panel.add_key_action('?', lambda : self.toggle_status(EventBuilder.STATUS_UNEXPECTED_SUCCESS), "Toggle showing/hiding tests whose status is 'unexpected_success'")
self.status_panel = lldbcurses.StatusPanel(frame=status_frame)
-
+
self.main_window.add_child(self.job_panel)
self.main_window.add_child(self.results_panel)
self.main_window.add_child(self.status_panel)
self.main_window.set_first_responder(self.results_panel)
-
+
self.status_panel.add_status_item(name="time", title="Elapsed", format="%s", width=20, value="0:00:00", update=False)
- self.status_panel.add_status_item(name="success", title="Success", format="%u", width=20, value=0, update=False)
- self.status_panel.add_status_item(name="failure", title="Failure", format="%u", width=20, value=0, update=False)
- self.status_panel.add_status_item(name="error", title="Error", format="%u", width=20, value=0, update=False)
- self.status_panel.add_status_item(name="skip", title="Skipped", format="%u", width=20, value=0, update=True)
- self.status_panel.add_status_item(name="expected_failure", title="Expected Failure", format="%u", width=30, value=0, update=False)
- self.status_panel.add_status_item(name="unexpected_success", title="Unexpected Success", format="%u", width=30, value=0, update=False)
+ self.status_panel.add_status_item(name=EventBuilder.STATUS_SUCCESS, title="Success", format="%u", width=20, value=0, update=False)
+ self.status_panel.add_status_item(name=EventBuilder.STATUS_FAILURE, title="Failure", format="%u", width=20, value=0, update=False)
+ self.status_panel.add_status_item(name=EventBuilder.STATUS_ERROR, title="Error", format="%u", width=20, value=0, update=False)
+ self.status_panel.add_status_item(name=EventBuilder.STATUS_SKIP, title="Skipped", format="%u", width=20, value=0, update=True)
+ self.status_panel.add_status_item(name=EventBuilder.STATUS_EXPECTED_FAILURE, title="Expected Failure", format="%u", width=30, value=0, update=False)
+ self.status_panel.add_status_item(name=EventBuilder.STATUS_UNEXPECTED_SUCCESS, title="Unexpected Success", format="%u", width=30, value=0, update=False)
self.main_window.refresh()
elif event == 'terminate':
#self.main_window.key_event_loop()
lldbcurses.terminate_curses()
check_for_one_key = False
self.using_terminal = False
- # Check for 1 keypress with no delay
-
+ # Check for 1 keypress with no delay
+
# Check for 1 keypress with no delay
if check_for_one_key:
- self.main_window.key_event_loop(0, 1)
-
+ self.main_window.key_event_loop(0, 1)
Modified: lldb/trunk/packages/Python/lldbsuite/test/dosep.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dosep.py?rev=254530&r1=254529&r2=254530&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/dosep.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/dosep.py Wed Dec 2 12:48:38 2015
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
"""
Run the test suite using a separate process for each test file.
@@ -55,6 +53,7 @@ import lldbsuite.support.seven as seven
from . import dotest_channels
from . import dotest_args
+from . import test_results
# Todo: Convert this folder layout to be relative-import friendly and don't hack up
# sys.path like this
@@ -1406,33 +1405,68 @@ def main(print_details_on_success, num_t
test_name = os.path.splitext(xtime)[0]
touch(os.path.join(session_dir, "{}-{}".format(result, test_name)))
- print()
- sys.stdout.write("Ran %d test suites" % num_test_files)
- if num_test_files > 0:
- sys.stdout.write(" (%d failed) (%f%%)" % (
- len(failed), 100.0 * len(failed) / num_test_files))
- print()
- sys.stdout.write("Ran %d test cases" % num_test_cases)
- if num_test_cases > 0:
- sys.stdout.write(" (%d failed) (%f%%)" % (
- fail_count, 100.0 * fail_count / num_test_cases))
- print()
- exit_code = 0
-
- if len(failed) > 0:
- failed.sort()
- print("Failing Tests (%d)" % len(failed))
- for f in failed:
- print("%s: LLDB (suite) :: %s (%s)" % (
- "TIMEOUT" if f in timed_out else "FAIL", f, system_info
- ))
- exit_code = 1
-
- if len(unexpected_successes) > 0:
- unexpected_successes.sort()
- print("\nUnexpected Successes (%d)" % len(unexpected_successes))
- for u in unexpected_successes:
- print("UNEXPECTED SUCCESS: LLDB (suite) :: %s (%s)" % (u, system_info))
+ # Only run the old summary logic if we don't have a results formatter
+ # that already prints the summary.
+ if results_formatter is None or not results_formatter.replaces_summary():
+ print_legacy_summary = True
+ else:
+ print_legacy_summary = False
+
+ if not print_legacy_summary:
+ # Remove this timeout handling once
+ # https://llvm.org/bugs/show_bug.cgi?id=25703
+ # is addressed.
+ #
+ # Use non-event-based structures to count timeouts.
+ timeout_count = len(timed_out)
+ if timeout_count > 0:
+ failed.sort()
+ print("Timed out test files: {}".format(len(timed_out)))
+ for f in failed:
+ if f in timed_out:
+ print("TIMEOUT: %s (%s)" % (f, system_info))
+
+ # Figure out exit code by count of test result types.
+ issue_count = (
+ results_formatter.counts_by_test_result_status(
+ test_results.EventBuilder.STATUS_ERROR) +
+ results_formatter.counts_by_test_result_status(
+ test_results.EventBuilder.STATUS_FAILURE) +
+ timeout_count)
+ # Return with appropriate result code
+ if issue_count > 0:
+ sys.exit(1)
+ else:
+ sys.exit(0)
+ else:
+ # Print the legacy test results summary.
+ print()
+ sys.stdout.write("Ran %d test suites" % num_test_files)
+ if num_test_files > 0:
+ sys.stdout.write(" (%d failed) (%f%%)" % (
+ len(failed), 100.0 * len(failed) / num_test_files))
+ print()
+ sys.stdout.write("Ran %d test cases" % num_test_cases)
+ if num_test_cases > 0:
+ sys.stdout.write(" (%d failed) (%f%%)" % (
+ fail_count, 100.0 * fail_count / num_test_cases))
+ print()
+ exit_code = 0
+
+ if len(failed) > 0:
+ failed.sort()
+ print("Failing Tests (%d)" % len(failed))
+ for f in failed:
+ print("%s: LLDB (suite) :: %s (%s)" % (
+ "TIMEOUT" if f in timed_out else "FAIL", f, system_info
+ ))
+ exit_code = 1
+
+ if len(unexpected_successes) > 0:
+ unexpected_successes.sort()
+ print("\nUnexpected Successes (%d)" % len(unexpected_successes))
+ for u in unexpected_successes:
+ print("UNEXPECTED SUCCESS: LLDB (suite) :: %s (%s)" % (u, system_info))
sys.exit(exit_code)
Modified: lldb/trunk/packages/Python/lldbsuite/test/test_results.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/test_results.py?rev=254530&r1=254529&r2=254530&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/test_results.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/test_results.py Wed Dec 2 12:48:38 2015
@@ -29,11 +29,20 @@ from six.moves import cPickle
# LLDB modules
+
class EventBuilder(object):
"""Helper class to build test result event dictionaries."""
BASE_DICTIONARY = None
+ # Test Status Tags
+ STATUS_SUCCESS = "success"
+ STATUS_FAILURE = "failure"
+ STATUS_EXPECTED_FAILURE = "expected_failure"
+ STATUS_UNEXPECTED_SUCCESS = "unexpected_success"
+ STATUS_SKIP = "skip"
+ STATUS_ERROR = "error"
+
@staticmethod
def _get_test_name_info(test):
"""Returns (test-class-name, test-method-name) from a test case instance.
@@ -182,7 +191,8 @@ class EventBuilder(object):
@return the event dictionary
"""
- return EventBuilder._event_dictionary_test_result(test, "success")
+ return EventBuilder._event_dictionary_test_result(
+ test, EventBuilder.STATUS_SUCCESS)
@staticmethod
def event_for_unexpected_success(test, bugnumber):
@@ -199,7 +209,7 @@ class EventBuilder(object):
"""
event = EventBuilder._event_dictionary_test_result(
- test, "unexpected_success")
+ test, EventBuilder.STATUS_UNEXPECTED_SUCCESS)
if bugnumber:
event["bugnumber"] = str(bugnumber)
return event
@@ -216,7 +226,7 @@ class EventBuilder(object):
@return the event dictionary
"""
return EventBuilder._event_dictionary_issue(
- test, "failure", error_tuple)
+ test, EventBuilder.STATUS_FAILURE, error_tuple)
@staticmethod
def event_for_expected_failure(test, error_tuple, bugnumber):
@@ -234,7 +244,7 @@ class EventBuilder(object):
"""
event = EventBuilder._event_dictionary_issue(
- test, "expected_failure", error_tuple)
+ test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple)
if bugnumber:
event["bugnumber"] = str(bugnumber)
return event
@@ -249,7 +259,8 @@ class EventBuilder(object):
@return the event dictionary
"""
- event = EventBuilder._event_dictionary_test_result(test, "skip")
+ event = EventBuilder._event_dictionary_test_result(
+ test, EventBuilder.STATUS_SKIP)
event["skip_reason"] = reason
return event
@@ -264,7 +275,8 @@ class EventBuilder(object):
@return the event dictionary
"""
- return EventBuilder._event_dictionary_issue(test, "error", error_tuple)
+ return EventBuilder._event_dictionary_issue(
+ test, EventBuilder.STATUS_ERROR, error_tuple)
@staticmethod
def event_for_cleanup_error(test, error_tuple):
@@ -279,7 +291,7 @@ class EventBuilder(object):
@return the event dictionary
"""
event = EventBuilder._event_dictionary_issue(
- test, "error", error_tuple)
+ test, EventBuilder.STATUS_ERROR, error_tuple)
event["issue_phase"] = "cleanup"
return event
@@ -376,7 +388,6 @@ class ResultsFormatter(object):
expectations about when the call should be chained.
"""
-
@classmethod
def arg_parser(cls):
"""@return arg parser used to parse formatter-specific options."""
@@ -396,6 +407,16 @@ class ResultsFormatter(object):
self.start_time_by_test = {}
self.terminate_called = False
+ # Store counts of test_result events by status.
+ self.result_status_counts = {
+ EventBuilder.STATUS_SUCCESS: 0,
+ EventBuilder.STATUS_EXPECTED_FAILURE: 0,
+ EventBuilder.STATUS_SKIP: 0,
+ EventBuilder.STATUS_UNEXPECTED_SUCCESS: 0,
+ EventBuilder.STATUS_FAILURE: 0,
+ EventBuilder.STATUS_ERROR: 0
+ }
+
# Lock that we use while mutating inner state, like the
# total test count and the elements. We minimize how
# long we hold the lock just to keep inner state safe, not
@@ -417,14 +438,19 @@ class ResultsFormatter(object):
# atexit() cleanup can call the "terminate if it hasn't been
# called yet".
if test_event is not None:
- if test_event.get("event", "") == "terminate":
+ event_type = test_event.get("event", "")
+ if event_type == "terminate":
self.terminate_called = True
+ elif event_type == "test_result":
+ # Keep track of event counts per test result status type
+ status = test_event["status"]
+ self.result_status_counts[status] += 1
def track_start_time(self, test_class, test_name, start_time):
- """Tracks the start time of a test so elapsed time can be computed.
+ """tracks the start time of a test so elapsed time can be computed.
- This alleviates the need for test results to be processed serially
- by test. It will save the start time for the test so that
+ this alleviates the need for test results to be processed serially
+ by test. it will save the start time for the test so that
elapsed_time_for_test() can compute the elapsed time properly.
"""
if test_class is None or test_name is None:
@@ -435,9 +461,9 @@ class ResultsFormatter(object):
self.start_time_by_test[test_key] = start_time
def elapsed_time_for_test(self, test_class, test_name, end_time):
- """Returns the elapsed time for a test.
+ """returns the elapsed time for a test.
- This function can only be called once per test and requires that
+ this function can only be called once per test and requires that
the track_start_time() method be called sometime prior to calling
this method.
"""
@@ -454,16 +480,38 @@ class ResultsFormatter(object):
return end_time - start_time
def is_using_terminal(self):
- """Returns True if this results formatter is using the terminal and
+ """returns true if this results formatter is using the terminal and
output should be avoided."""
return self.using_terminal
def send_terminate_as_needed(self):
- """Sends the terminate event if it hasn't been received yet."""
+ """sends the terminate event if it hasn't been received yet."""
if not self.terminate_called:
terminate_event = EventBuilder.bare_event("terminate")
self.handle_event(terminate_event)
+ # Derived classes may require self access
+ # pylint: disable=no-self-use
+ def replaces_summary(self):
+ """Returns whether the results formatter includes a summary
+ suitable to replace the old lldb test run results.
+
+ @return True if the lldb test runner can skip its summary
+ generation when using this results formatter; False otherwise.
+ """
+ return False
+
+ def counts_by_test_result_status(self, status):
+ """Returns number of test method results for the given status.
+
+ @status_result a test result status (e.g. success, fail, skip)
+ as defined by the EventBuilder.STATUS_* class members.
+
+ @return an integer returning the number of test methods matching
+ the given test result status.
+ """
+ return self.result_status_counts[status]
+
class XunitFormatter(ResultsFormatter):
"""Provides xUnit-style formatted output.
@@ -527,7 +575,8 @@ class XunitFormatter(ResultsFormatter):
unicode_content = str_or_unicode.decode('utf-8')
else:
unicode_content = str_or_unicode
- return self.invalid_xml_re.sub(six.u('?'), unicode_content).encode('utf-8')
+ return self.invalid_xml_re.sub(
+ six.u('?'), unicode_content).encode('utf-8')
@classmethod
def arg_parser(cls):
@@ -622,12 +671,14 @@ class XunitFormatter(ResultsFormatter):
}
self.status_handlers = {
- "success": self._handle_success,
- "failure": self._handle_failure,
- "error": self._handle_error,
- "skip": self._handle_skip,
- "expected_failure": self._handle_expected_failure,
- "unexpected_success": self._handle_unexpected_success
+ EventBuilder.STATUS_SUCCESS: self._handle_success,
+ EventBuilder.STATUS_FAILURE: self._handle_failure,
+ EventBuilder.STATUS_ERROR: self._handle_error,
+ EventBuilder.STATUS_SKIP: self._handle_skip,
+ EventBuilder.STATUS_EXPECTED_FAILURE:
+ self._handle_expected_failure,
+ EventBuilder.STATUS_UNEXPECTED_SUCCESS:
+ self._handle_unexpected_success
}
def handle_event(self, test_event):
More information about the lldb-commits
mailing list