[Lldb-commits] [lldb] r240789 - Added expectedFlakey test decorator
Vince Harron
vince at nethacker.com
Fri Jun 26 08:13:21 PDT 2015
Author: vharron
Date: Fri Jun 26 10:13:21 2015
New Revision: 240789
URL: http://llvm.org/viewvc/llvm-project?rev=240789&view=rev
Log:
Added expectedFlakey test decorator
SUMMARY
Flakey tests get two chances to pass
Also, switched a bunch of tests to use new decorator.
TEST PLAN
Add one of these decorators to a test
Edit a test to pass on the first invocation, confirm test appears as pass
Edit a test to pass on the first invocation, pass on the second, confirm test appears as xfail
Edit a test to fail on two consecutive runs, confirm test appears in results as fail/error
Differential Revision: http://reviews.llvm.org/D10721
Modified:
lldb/trunk/test/expression_command/call-function/TestCallStopAndContinue.py
lldb/trunk/test/expression_command/call-restarts/TestCallThatRestarts.py
lldb/trunk/test/expression_command/timeout/TestCallWithTimeout.py
lldb/trunk/test/functionalities/attach_resume/TestAttachResume.py
lldb/trunk/test/functionalities/avoids-fd-leak/TestFdLeak.py
lldb/trunk/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py
lldb/trunk/test/functionalities/stop-hook/TestStopHookMechanism.py
lldb/trunk/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py
lldb/trunk/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
lldb/trunk/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
lldb/trunk/test/lang/c/register_variables/TestRegisterVariables.py
lldb/trunk/test/lang/cpp/this/TestCPPThis.py
lldb/trunk/test/lldbtest.py
lldb/trunk/test/python_api/signals/TestSignalsAPI.py
lldb/trunk/test/types/TestFloatTypes.py
Modified: lldb/trunk/test/expression_command/call-function/TestCallStopAndContinue.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/expression_command/call-function/TestCallStopAndContinue.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/expression_command/call-function/TestCallStopAndContinue.py (original)
+++ lldb/trunk/test/expression_command/call-function/TestCallStopAndContinue.py Fri Jun 26 10:13:21 2015
@@ -22,14 +22,14 @@ class ExprCommandCallStopContinueTestCas
@skipUnlessDarwin
@dsym_test
- @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX
+ @expectedFlakeyDarwin("llvm.org/pr20274")
def test_with_dsym(self):
"""Test gathering result from interrupted function call."""
self.buildDsym()
self.call_function()
@dwarf_test
- @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX
+ @expectedFlakeyDarwin("llvm.org/pr20274")
def test_with_dwarf(self):
"""Test gathering result from interrupted function call."""
self.buildDwarf()
Modified: lldb/trunk/test/expression_command/call-restarts/TestCallThatRestarts.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/expression_command/call-restarts/TestCallThatRestarts.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/expression_command/call-restarts/TestCallThatRestarts.py (original)
+++ lldb/trunk/test/expression_command/call-restarts/TestCallThatRestarts.py Fri Jun 26 10:13:21 2015
@@ -29,7 +29,7 @@ class ExprCommandThatRestartsTestCase(Te
@dwarf_test
@skipIfFreeBSD # llvm.org/pr19246: intermittent failure
- @expectedFailureLinux("llvm.org/pr19246") # intermittent failure
+ @expectedFailureLinux("llvm.org/pr19246")
@skipIfDarwin # llvm.org/pr19246: intermittent failure
@skipIfWindows # Test relies on signals, unsupported on Windows
def test_with_dwarf(self):
Modified: lldb/trunk/test/expression_command/timeout/TestCallWithTimeout.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/expression_command/timeout/TestCallWithTimeout.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/expression_command/timeout/TestCallWithTimeout.py (original)
+++ lldb/trunk/test/expression_command/timeout/TestCallWithTimeout.py Fri Jun 26 10:13:21 2015
@@ -28,7 +28,7 @@ class ExprCommandWithTimeoutsTestCase(Te
self.call_function()
@expectedFailureFreeBSD("llvm.org/pr19605") # fails on buildbot
- @expectedFailureLinux("llvm.org/pr20275") # fails intermittently on Linux
+ @expectedFlakeyLinux("llvm.org/pr20275")
@dwarf_test
def test_with_dwarf(self):
"""Test calling std::String member function."""
Modified: lldb/trunk/test/functionalities/attach_resume/TestAttachResume.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/functionalities/attach_resume/TestAttachResume.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/functionalities/attach_resume/TestAttachResume.py (original)
+++ lldb/trunk/test/functionalities/attach_resume/TestAttachResume.py Fri Jun 26 10:13:21 2015
@@ -22,7 +22,7 @@ class AttachResumeTestCase(TestBase):
self.buildDwarf()
self.process_attach_continue_interrupt_detach()
- @expectedFailureLinux('llvm.org/pr19478') # intermittent ~2/14 runs
+ @expectedFlakeyLinux('llvm.org/pr19478') # intermittent ~2/14 runs
@skipIfRemote
def process_attach_continue_interrupt_detach(self):
"""Test attach/continue/interrupt/detach"""
Modified: lldb/trunk/test/functionalities/avoids-fd-leak/TestFdLeak.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/functionalities/avoids-fd-leak/TestFdLeak.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/functionalities/avoids-fd-leak/TestFdLeak.py (original)
+++ lldb/trunk/test/functionalities/avoids-fd-leak/TestFdLeak.py Fri Jun 26 10:13:21 2015
@@ -41,8 +41,8 @@ class AvoidsFdLeakTestCase(TestBase):
self.assertTrue(process.GetExitStatus() == 0,
"Process returned non-zero status. Were incorrect file descriptors passed?")
- @expectedFailure(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd
- @expectedFailureLinux # xfail flakey test to get buildbot green
+ @expectedFlakey(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd
+ @expectedFlakeyLinux
@skipIfWindows # The check for descriptor leakage needs to be implemented differently here.
@skipIfTargetAndroid() # Android have some other file descriptors open by the shell
def test_fd_leak_multitarget (self):
Modified: lldb/trunk/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py (original)
+++ lldb/trunk/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py Fri Jun 26 10:13:21 2015
@@ -24,7 +24,7 @@ class ExprDoesntDeadlockTestCase(TestBas
@dwarf_test
@expectedFailureFreeBSD('llvm.org/pr17946')
- @expectedFailureLinux # failed 1/365 test runs, line 61, thread.IsValid()
+ @expectedFlakeyLinux # failed 1/365 test runs, line 61, thread.IsValid()
def test_with_dwarf_and_run_command(self):
"""Test that expr will time out and allow other threads to run if it blocks."""
self.buildDwarf()
Modified: lldb/trunk/test/functionalities/stop-hook/TestStopHookMechanism.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/functionalities/stop-hook/TestStopHookMechanism.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/functionalities/stop-hook/TestStopHookMechanism.py (original)
+++ lldb/trunk/test/functionalities/stop-hook/TestStopHookMechanism.py Fri Jun 26 10:13:21 2015
@@ -19,7 +19,7 @@ class StopHookMechanismTestCase(TestBase
self.stop_hook_firing()
@skipIfFreeBSD # llvm.org/pr15037
- @expectedFailureLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux
+ @expectedFlakeyLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux
@expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
@dwarf_test
def test_with_dwarf(self):
Modified: lldb/trunk/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py (original)
+++ lldb/trunk/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py Fri Jun 26 10:13:21 2015
@@ -21,7 +21,7 @@ class StopHookForMultipleThreadsTestCase
@dwarf_test
@expectedFailureFreeBSD("llvm.org/pr15037")
- @expectedFailureLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux
+ @expectedFlakeyLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux
@expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
@skipIfFreeBSD # llvm.org/pr22784
def test_stop_hook_multiple_threads_with_dwarf(self):
Modified: lldb/trunk/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py (original)
+++ lldb/trunk/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py Fri Jun 26 10:13:21 2015
@@ -22,6 +22,8 @@ class CreateAfterAttachTestCase(TestBase
@skipIfFreeBSD # Hangs. May be the same as Linux issue llvm.org/pr16229 but
# not yet investigated. Revisit once required functionality
# is implemented for FreeBSD.
+ @skipIfLinux # Occasionally hangs on the build bot, expectedFailureLinux
+
@dwarf_test
def test_create_after_attach_with_dwarf_and_popen(self):
"""Test thread creation after process attach."""
@@ -32,6 +34,7 @@ class CreateAfterAttachTestCase(TestBase
# for FreeBSD.
@dwarf_test
@skipIfRemote
+ @expectedFlakeyLinux("llvm.org/pr16229") # 1/100 dosep, build 3546, clang-3.5 x84_64
def test_create_after_attach_with_dwarf_and_fork(self):
"""Test thread creation after process attach."""
self.buildDwarf(dictionary=self.getBuildFlags(use_cpp11=False))
Modified: lldb/trunk/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py (original)
+++ lldb/trunk/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py Fri Jun 26 10:13:21 2015
@@ -23,7 +23,7 @@ class ThreadSpecificBreakTestCase(TestBa
@expectedFailureFreeBSD('llvm.org/pr18522') # hits break in another thread in testrun
@python_api_test
@dwarf_test
- @expectedFailureLinux # this test fails 6/100 dosep runs
+ @expectedFlakeyLinux # this test fails 6/100 dosep runs
def test_with_dwarf_python(self):
"""Test that we obey thread conditioned breakpoints."""
self.buildDwarf()
Modified: lldb/trunk/test/lang/c/register_variables/TestRegisterVariables.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/lang/c/register_variables/TestRegisterVariables.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/lang/c/register_variables/TestRegisterVariables.py (original)
+++ lldb/trunk/test/lang/c/register_variables/TestRegisterVariables.py Fri Jun 26 10:13:21 2015
@@ -19,7 +19,7 @@ class RegisterVariableTestCase(TestBase)
@dwarf_test
@expectedFailureClang(None, ['<', '3.5'])
- @expectedFailureGcc #xfail to get buildbot green, test failed with gcc4.8.2
+ @expectedFlakeyGcc # failed with gcc4.8.2
def test_with_dwarf_and_run_command(self):
"""Test expressions on register values."""
self.buildDwarf()
Modified: lldb/trunk/test/lang/cpp/this/TestCPPThis.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/lang/cpp/this/TestCPPThis.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/lang/cpp/this/TestCPPThis.py (original)
+++ lldb/trunk/test/lang/cpp/this/TestCPPThis.py Fri Jun 26 10:13:21 2015
@@ -22,7 +22,7 @@ class CPPThisTestCase(TestBase):
@expectedFailureGcc # llvm.org/pr15439 The 'this' pointer isn't available during expression evaluation when stopped in an inlined member function.
@expectedFailureIcc # ICC doesn't emit correct DWARF inline debug info for inlined member functions
@dwarf_test
- @expectedFailureClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6'])#xfail to get buildbot green, test failed with totclang - clang3.7
+ @expectedFlakeyClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6']) # failed with totclang - clang3.7
def test_with_dwarf_and_run_command(self):
"""Test that the appropriate member variables are available when stopped in C++ static, inline, and const methods"""
self.buildDwarf()
Modified: lldb/trunk/test/lldbtest.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/lldbtest.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/lldbtest.py (original)
+++ lldb/trunk/test/lldbtest.py Fri Jun 26 10:13:21 2015
@@ -712,6 +712,68 @@ def expectedFailureAndroid(bugnumber=Non
return expectedFailure(fn, bugnumber)
+# if the test passes on the first try, we're done (success)
+# if the test fails once, then passes on the second try, raise an ExpectedFailure
+# if the test fails twice in a row, re-throw the exception from the second test run
+def expectedFlakey(expected_fn, bugnumber=None):
+ def expectedFailure_impl(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ from unittest2 import case
+ self = args[0]
+ try:
+ func(*args, **kwargs)
+ except Exception:
+ if expected_fn(self):
+ # retry
+ try:
+ func(*args, **kwargs)
+ except Exception:
+ # oh snap! two failures in a row, record a failure/error
+ raise
+ # record the expected failure
+ raise case._ExpectedFailure(sys.exc_info(), bugnumber)
+ else:
+ raise
+ return wrapper
+ # if bugnumber is not-callable(incluing None), that means decorator function is called with optional arguments
+ # return decorator in this case, so it will be used to decorating original method
+ if callable(bugnumber):
+ return expectedFailure_impl(bugnumber)
+ else:
+ return expectedFailure_impl
+
+def expectedFlakeyOS(oslist, bugnumber=None, compilers=None):
+ def fn(self):
+ return (self.getPlatform() in oslist and
+ self.expectedCompiler(compilers))
+ return expectedFlakey(fn, bugnumber)
+
+def expectedFlakeyDarwin(bugnumber=None, compilers=None):
+ # For legacy reasons, we support both "darwin" and "macosx" as OS X triples.
+ return expectedFlakeyOS(getDarwinOSTriples(), bugnumber, compilers)
+
+def expectedFlakeyLinux(bugnumber=None, compilers=None):
+ return expectedFlakeyOS(['linux'], bugnumber, compilers)
+
+def expectedFlakeyFreeBSD(bugnumber=None, compilers=None):
+ return expectedFlakeyOS(['freebsd'], bugnumber, compilers)
+
+def expectedFlakeyCompiler(compiler, compiler_version=None, bugnumber=None):
+ if compiler_version is None:
+ compiler_version=['=', None]
+ def fn(self):
+ return compiler in self.getCompiler() and self.expectedCompilerVersion(compiler_version)
+ return expectedFlakey(fn, bugnumber)
+
+# @expectedFlakeyClang('bugnumber', ['<=', '3.4'])
+def expectedFlakeyClang(bugnumber=None, compiler_version=None):
+ return expectedFlakeyCompiler('clang', compiler_version, bugnumber)
+
+# @expectedFlakeyGcc('bugnumber', ['<=', '3.4'])
+def expectedFlakeyGcc(bugnumber=None, compiler_version=None):
+ return expectedFlakeyCompiler('gcc', compiler_version, bugnumber)
+
def skipIfRemote(func):
"""Decorate the item to skip tests if testing remotely."""
if isinstance(func, type) and issubclass(func, unittest2.TestCase):
Modified: lldb/trunk/test/python_api/signals/TestSignalsAPI.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/python_api/signals/TestSignalsAPI.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/python_api/signals/TestSignalsAPI.py (original)
+++ lldb/trunk/test/python_api/signals/TestSignalsAPI.py Fri Jun 26 10:13:21 2015
@@ -12,7 +12,7 @@ class SignalsAPITestCase(TestBase):
mydir = TestBase.compute_mydir(__file__)
@python_api_test
- @expectedFailureLinux # this test fails 1/100 dosep runs
+ @expectedFlakeyLinux # this test fails 1/100 dosep runs
def test_ignore_signal(self):
"""Test Python SBUnixSignals.Suppress/Stop/Notify() API."""
self.buildDefault()
Modified: lldb/trunk/test/types/TestFloatTypes.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/test/types/TestFloatTypes.py?rev=240789&r1=240788&r2=240789&view=diff
==============================================================================
--- lldb/trunk/test/types/TestFloatTypes.py (original)
+++ lldb/trunk/test/types/TestFloatTypes.py Fri Jun 26 10:13:21 2015
@@ -38,7 +38,7 @@ class FloatTypesTestCase(AbstractBase.Ge
@skipUnlessDarwin
@dsym_test
- @expectedFailureDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110
+ @expectedFlakeyDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110
def test_double_type_with_dsym(self):
"""Test that double-type variables are displayed correctly."""
self.build_and_run('double.cpp', set(['double']))
More information about the lldb-commits
mailing list