[llvm] r246704 - [lit] Add basic flaky test retry functionality

[Reid Kleckner via llvm-commits]

Author: rnk
Date: Wed Sep  2 15:32:41 2015
New Revision: 246704

URL: http://llvm.org/viewvc/llvm-project?rev=246704&view=rev
Log:
[lit] Add basic flaky test retry functionality

The plan is to use this for the sanitizer test suite on Windows.  See
PR24554 for more details on why we need this.

Tested manually by injecting rand() into a sanitizer test and watching
what it does.

Modified:
    llvm/trunk/utils/lit/lit/Test.py
    llvm/trunk/utils/lit/lit/TestRunner.py
    llvm/trunk/utils/lit/lit/main.py

Modified: llvm/trunk/utils/lit/lit/Test.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/Test.py?rev=246704&r1=246703&r2=246704&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/Test.py (original)
+++ llvm/trunk/utils/lit/lit/Test.py Wed Sep  2 15:32:41 2015
@@ -27,6 +27,7 @@ class ResultCode(object):
                          (self.name, self.isFailure))
 
 PASS        = ResultCode('PASS', False)
+FLAKYPASS   = ResultCode('FLAKYPASS', False)
 XFAIL       = ResultCode('XFAIL', False)
 FAIL        = ResultCode('FAIL', True)
 XPASS       = ResultCode('XPASS', True)
@@ -253,4 +254,4 @@ class Test:
             xml += "\n\t</failure>\n</testcase>"
         else:
             xml += "/>"
-        return xml
\ No newline at end of file
+        return xml

Modified: llvm/trunk/utils/lit/lit/TestRunner.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/TestRunner.py?rev=246704&r1=246703&r2=246704&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/TestRunner.py (original)
+++ llvm/trunk/utils/lit/lit/TestRunner.py Wed Sep  2 15:32:41 2015
@@ -602,5 +602,17 @@ def executeShTest(test, litConfig, useEx
         return lit.Test.Result(Test.PASS)
 
     script, tmpBase, execdir = res
-    return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
 
+    # Re-run failed tests up to test_retry_attempts times.
+    attempts = 1
+    if hasattr(test.config, 'test_retry_attempts'):
+        attempts += test.config.test_retry_attempts
+    for i in range(attempts):
+        res = _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
+        if res.code != Test.FAIL:
+            break
+    # If we had to run the test more than once, count it as a flaky pass. These
+    # will be printed separately in the test summary.
+    if i > 0 and res.code == Test.PASS:
+        res.code = Test.FLAKYPASS
+    return res

Modified: llvm/trunk/utils/lit/lit/main.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/main.py?rev=246704&r1=246703&r2=246704&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/main.py (original)
+++ llvm/trunk/utils/lit/lit/main.py Wed Sep  2 15:32:41 2015
@@ -414,6 +414,7 @@ def main(builtinParameters = {}):
         lit.util.printHistogram(test_times, title='Tests')
 
     for name,code in (('Expected Passes    ', lit.Test.PASS),
+                      ('Passes With Retry  ', lit.Test.FLAKYPASS),
                       ('Expected Failures  ', lit.Test.XFAIL),
                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),