[PATCH] D77819: [lit] Add SKIPPED test result category

[Julian Lettner via Phabricator via llvm-commits]

yln created this revision.
yln added reviewers: rnk, jdenny, serge-sans-paille, probinson.
Herald added subscribers: llvm-commits, delcypher.
Herald added a project: LLVM.

Track and print the number of skipped tests on user interrupts [Ctrl+C].
This is part of a larger effort to ensure that all discovered tests are
properly accounted for.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D77819

Files:
  llvm/utils/lit/lit/Test.py
  llvm/utils/lit/lit/main.py
  llvm/utils/lit/lit/run.py


Index: llvm/utils/lit/lit/run.py
===================================================================
--- llvm/utils/lit/lit/run.py
+++ llvm/utils/lit/lit/run.py
@@ -42,7 +42,7 @@
 
         Upon completion, each test in the run will have its result
         computed. Tests which were not actually executed (for any reason) will
-        be given an UNRESOLVED result.
+        be marked SKIPPED.
         """
         self.failures = 0
 
@@ -51,12 +51,13 @@
         timeout = self.timeout or one_week
         deadline = time.time() + timeout
 
-        self._execute(deadline)
-
-        # Mark any tests that weren't run as UNRESOLVED.
-        for test in self.tests:
-            if test.result is None:
-                test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
+        try:
+            self._execute(deadline)
+        finally:
+            skipped = lit.Test.Result(lit.Test.SKIPPED)
+            for test in self.tests:
+                if test.result is None:
+                    test.setResult(skipped)
 
     def _execute(self, deadline):
         self._increase_process_limit()
Index: llvm/utils/lit/lit/main.py
===================================================================
--- llvm/utils/lit/lit/main.py
+++ llvm/utils/lit/lit/main.py
@@ -89,12 +89,14 @@
     run_tests(filtered_tests, lit_config, opts, len(discovered_tests))
     elapsed = time.time() - start
 
-    executed_tests = [t for t in filtered_tests if t.result]
+    # TODO(yln): eventually, all functions below should act on discovered_tests
+    executed_tests = [
+        t for t in filtered_tests if t.result.code != lit.Test.SKIPPED]
 
     if opts.time_tests:
         print_histogram(executed_tests)
 
-    print_results(executed_tests, elapsed, opts)
+    print_results(filtered_tests, elapsed, opts)
 
     if opts.output_path:
         #TODO(yln): pass in discovered_tests
@@ -256,6 +258,7 @@
 ]
 
 all_codes = [
+    (lit.Test.SKIPPED,     'Skipped Tests',     'Skipped'),
     (lit.Test.UNSUPPORTED, 'Unsupported Tests', 'Unsupported'),
     (lit.Test.PASS,        'Expected Passes',   ''),
     (lit.Test.FLAKYPASS,   'Passes With Retry', ''),
@@ -277,11 +280,11 @@
 def print_group(code, label, tests, opts):
     if not tests:
         return
-    if code == lit.Test.PASS:
+    # TODO(yln): FLAKYPASS? Make this more consistent!
+    if code in {lit.Test.SKIPPED, lit.Test.PASS}:
         return
     if (lit.Test.XFAIL == code and not opts.show_xfail) or \
-       (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
-       (lit.Test.UNRESOLVED == code and (opts.max_failures is not None)):
+       (lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
         return
     print('*' * 20)
     print('%s Tests (%d):' % (label, len(tests)))
Index: llvm/utils/lit/lit/Test.py
===================================================================
--- llvm/utils/lit/lit/Test.py
+++ llvm/utils/lit/lit/Test.py
@@ -36,6 +36,7 @@
 UNRESOLVED  = ResultCode('UNRESOLVED', True)
 UNSUPPORTED = ResultCode('UNSUPPORTED', False)
 TIMEOUT     = ResultCode('TIMEOUT', True)
+SKIPPED     = ResultCode('SKIPPED', False)
 
 # Test metric values.
 


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D77819.256347.patch
Type: text/x-patch
Size: 3180 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20200409/d1f5bdc4/attachment.bin>