[Lldb-commits] [lldb] r255763 - [test] Add ability to expect timeouts
Pavel Labath via lldb-commits
lldb-commits at lists.llvm.org
Wed Dec 16 04:09:52 PST 2015
Author: labath
Date: Wed Dec 16 06:09:45 2015
New Revision: 255763
URL: http://llvm.org/viewvc/llvm-project?rev=255763&view=rev
Log:
[test] Add ability to expect timeouts
Summary:
This adds ability to mark test that do not complete due to hangs, crashes, etc., as "expected",
to avoid flagging the build red for a known problem. Functionally, this extends the scope of the
existing expectedFailureXXX decorators to cover these states as well. Once this is in, I will
start replacing the magic list of failing tests in dosep.py with our regular annotations which
should hopefully make code simpler.
Reviewers: tfiala
Subscribers: lldb-commits
Differential Revision: http://reviews.llvm.org/D15530
Modified:
lldb/trunk/packages/Python/lldbsuite/test/dosep.py
lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py
lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py
Modified: lldb/trunk/packages/Python/lldbsuite/test/dosep.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dosep.py?rev=255763&r1=255762&r2=255763&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/dosep.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/dosep.py Wed Dec 16 06:09:45 2015
@@ -1143,7 +1143,6 @@ def getExpectedTimeouts(platform_name):
target = m.group(1)
expected_timeout = set()
- expected_timeout.add("TestExpectedTimeout.py")
if target.startswith("linux"):
expected_timeout |= {
Modified: lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park?rev=255763&r1=255762&r2=255763&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park Wed Dec 16 06:09:45 2015
@@ -10,6 +10,7 @@ class ExpectedTimeoutTestCase(lldbtest.T
"""Forces test timeout."""
mydir = lldbtest.TestBase.compute_mydir(__file__)
+ @lldbtest.expectedFailureAll()
def test_buildbot_sees_expected_timeout(self):
"""Tests that expected timeout logic kicks in and is picked up."""
while True:
Modified: lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py?rev=255763&r1=255762&r2=255763&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py Wed Dec 16 06:09:45 2015
@@ -598,6 +598,10 @@ def expectedFailure(expected_fn, bugnumb
from unittest2 import case
self = args[0]
if expected_fn(self):
+ if configuration.results_formatter_object is not None:
+ # Mark this test as expected to fail.
+ configuration.results_formatter_object.handle_event(
+ EventBuilder.event_for_mark_test_expected_failure(self))
xfail_func = unittest2.expectedFailure(func)
xfail_func(*args, **kwargs)
else:
Modified: lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py?rev=255763&r1=255762&r2=255763&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py Wed Dec 16 06:09:45 2015
@@ -165,6 +165,7 @@ class EventBuilder(object):
TYPE_TEST_RESULT = "test_result"
TYPE_TEST_START = "test_start"
TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
+ TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure"
TYPE_SESSION_TERMINATE = "terminate"
RESULT_TYPES = set([
@@ -528,6 +529,20 @@ class EventBuilder(object):
return event
@staticmethod
+ def event_for_mark_test_expected_failure(test):
+ """Creates an event that indicates the specified test is expected
+ to fail.
+
+ @param test the TestCase instance to which this pertains.
+
+ @return an event that specifies the given test is expected to fail.
+ """
+ event = EventBuilder._event_dictionary_common(
+ test,
+ EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE)
+ return event
+
+ @staticmethod
def add_entries_to_all_events(entries_dict):
"""Specifies a dictionary of entries to add to all test events.
@@ -681,6 +696,11 @@ class ResultsFormatter(object):
# timeout test status for this.
self.expected_timeouts_by_basename = set()
+ # Tests which have reported that they are expecting to fail. These will
+ # be marked as expected failures even if they return a failing status,
+ # probably because they crashed or deadlocked.
+ self.expected_failures = set()
+
# Keep track of rerun-eligible tests.
# This is a set that contains tests saved as:
# {test_filename}:{test_class}:{test_name}
@@ -721,6 +741,15 @@ class ResultsFormatter(object):
component_count += 1
return key
+ def _mark_test_as_expected_failure(self, test_result_event):
+ key = self._make_key(test_result_event)
+ if key is not None:
+ self.expected_failures.add(key)
+ else:
+ sys.stderr.write(
+ "\nerror: test marked as expected failure but "
+ "failed to create key.\n")
+
def _mark_test_for_rerun_eligibility(self, test_result_event):
key = self._make_key(test_result_event)
if key is not None:
@@ -796,6 +825,20 @@ class ResultsFormatter(object):
# Convert to an expected timeout.
event["status"] = EventBuilder.STATUS_EXPECTED_TIMEOUT
+ def _maybe_remap_expected_failure(self, event):
+ if event is None:
+ return
+
+ key = self._make_key(event)
+ if key not in self.expected_failures:
+ return
+
+ status = event.get("status", None)
+ if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES:
+ event["status"] = EventBuilder.STATUS_EXPECTED_FAILURE
+ elif status == EventBuilder.STATUS_SUCCESS:
+ event["status"] = EventBuilder.STATUS_UNEXPECTED_SUCCESS
+
def handle_event(self, test_event):
"""Handles the test event for collection into the formatter output.
@@ -824,6 +867,7 @@ class ResultsFormatter(object):
# Remap timeouts to expected timeouts.
if event_type in EventBuilder.RESULT_TYPES:
self._maybe_remap_expected_timeout(test_event)
+ self._maybe_remap_expected_failure(test_event)
event_type = test_event.get("event", "")
if event_type == "terminate":
@@ -887,6 +931,8 @@ class ResultsFormatter(object):
elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE:
self._mark_test_for_rerun_eligibility(test_event)
+ elif event_type == EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE:
+ self._mark_test_as_expected_failure(test_event)
def set_expected_timeouts_by_basename(self, basenames):
"""Specifies a list of test file basenames that are allowed to timeout
More information about the lldb-commits
mailing list