[llvm] 2ac96d6 - [lit] Improve consistency when printing test results

[Julian Lettner via llvm-commits]

Author: Julian Lettner
Date: 2020-04-07T18:12:18-07:00
New Revision: 2ac96d61c2ff21fca6dd22bf1b9545dbd450e4b9

URL: https://github.com/llvm/llvm-project/commit/2ac96d61c2ff21fca6dd22bf1b9545dbd450e4b9
DIFF: https://github.com/llvm/llvm-project/commit/2ac96d61c2ff21fca6dd22bf1b9545dbd450e4b9.diff

LOG: [lit] Improve consistency when printing test results

Added: 
    

Modified: 
    llvm/utils/lit/lit/main.py
    llvm/utils/lit/tests/shtest-format.py

Removed: 
    


################################################################################
diff  --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 3e5716f4a1ff..2930555269de 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -238,55 +238,59 @@ def execute_in_tmp_dir(run, lit_config):
                 # FIXME: Re-try after timeout on Windows.
                 lit_config.warning("Failed to delete temp directory '%s'" % tmp_dir)
 
-def print_summary(tests, elapsed, opts):
-    if not opts.quiet:
-        print('\nTesting Time: %.2fs' % elapsed)
 
-    byCode = {}
+def print_summary(tests, elapsed, opts):
+    # Status code, summary label, group label
+    groups = [
+        # Successes
+        (lit.Test.UNSUPPORTED, 'Unsupported Tests  ', 'Unsupported'),
+        (lit.Test.PASS,        'Expected Passes    ', ''),
+        (lit.Test.FLAKYPASS,   'Passes With Retry  ', ''),
+        (lit.Test.XFAIL,       'Expected Failures  ', 'Expected Failing'),
+        # Failures
+        (lit.Test.UNRESOLVED,  'Unresolved Tests   ', 'Unresolved'),
+        (lit.Test.TIMEOUT,     'Individual Timeouts', 'Timed Out'),
+        (lit.Test.FAIL,        'Unexpected Failures', 'Failing'),
+        (lit.Test.XPASS,       'Unexpected Passes  ', 'Unexpected Passing')]
+
+    by_code = {code: [] for (code, _, _) in groups}
     for test in tests:
-        if test.result.code not in byCode:
-            byCode[test.result.code] = []
-        byCode[test.result.code].append(test)
-
-    # Print each test in any of the failing groups.
-    for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
-                       ('Failing Tests', lit.Test.FAIL),
-                       ('Unresolved Tests', lit.Test.UNRESOLVED),
-                       ('Unsupported Tests', lit.Test.UNSUPPORTED),
-                       ('Expected Failing Tests', lit.Test.XFAIL),
-                       ('Timed Out Tests', lit.Test.TIMEOUT)):
-        if (lit.Test.XFAIL == code and not opts.show_xfail) or \
-           (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
-           (lit.Test.UNRESOLVED == code and (opts.max_failures is not None)):
-            continue
-        elts = byCode.get(code)
-        if not elts:
-            continue
-        print('*'*20)
-        print('%s (%d):' % (title, len(elts)))
-        for test in elts:
-            print('    %s' % test.getFullName())
-        sys.stdout.write('\n')
+        by_code[test.result.code].append(test)
+
+    for (code, _, group_label) in groups:
+        print_group(code, group_label, by_code[code], opts)
 
     if opts.timeTests and tests:
-        # Order by time.
-        test_times = [(test.getFullName(), test.result.elapsed)
-                      for test in tests]
+        test_times = [(t.getFullName(), t.result.elapsed) for t in tests]
         lit.util.printHistogram(test_times, title='Tests')
 
-    for name,code in (('Expected Passes    ', lit.Test.PASS),
-                      ('Passes With Retry  ', lit.Test.FLAKYPASS),
-                      ('Expected Failures  ', lit.Test.XFAIL),
-                      ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
-                      ('Unresolved Tests   ', lit.Test.UNRESOLVED),
-                      ('Unexpected Passes  ', lit.Test.XPASS),
-                      ('Unexpected Failures', lit.Test.FAIL),
-                      ('Individual Timeouts', lit.Test.TIMEOUT)):
-        if opts.quiet and not code.isFailure:
-            continue
-        N = len(byCode.get(code,[]))
-        if N:
-            print('  %s: %d' % (name,N))
+    if not opts.quiet:
+        print('\nTesting Time: %.2fs' % elapsed)
+    for (code, summary_label, _) in groups:
+        print_group_summary(code, summary_label, by_code[code], opts.quiet)
+
+
+def print_group(code, label, tests, opts):
+    if not tests:
+        return
+    if code == lit.Test.PASS:
+        return
+    if (lit.Test.XFAIL == code and not opts.show_xfail) or \
+       (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
+       (lit.Test.UNRESOLVED == code and (opts.max_failures is not None)):
+        return
+    print('*' * 20)
+    print('%s Tests (%d):' % (label, len(tests)))
+    for test in tests:
+        print('  %s' % test.getFullName())
+    sys.stdout.write('\n')
+
+
+def print_group_summary(code, label, tests, quiet):
+    count = len(tests)
+    if count and (code.isFailure or not quiet):
+        print('  %s: %d' % (label, count))
+
 
 def write_test_results(tests, lit_config, elapsed, output_path):
     # TODO(yln): audit: unexecuted tests

diff  --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 6ad4b0307023..75d08895f966 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -68,22 +68,22 @@
 # CHECK-NEXT: --
 # CHECK-NEXT: true
 # CHECK-NEXT: --
-# CHECK: Testing Time
-
-# CHECK: Unexpected Passing Tests (1)
-# CHECK: shtest-format :: xpass.txt
 
 # CHECK: Failing Tests (3)
 # CHECK: shtest-format :: external_shell/fail.txt
 # CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
 # CHECK: shtest-format :: fail.txt
 
+# CHECK: Unexpected Passing Tests (1)
+# CHECK: shtest-format :: xpass.txt
+
+# CHECK: Testing Time:
+# CHECK: Unsupported Tests  : 4
 # CHECK: Expected Passes    : 7
 # CHECK: Expected Failures  : 4
-# CHECK: Unsupported Tests  : 4
 # CHECK: Unresolved Tests   : 3
-# CHECK: Unexpected Passes  : 1
 # CHECK: Unexpected Failures: 3
+# CHECK: Unexpected Passes  : 1
 
 
 # XUNIT: <?xml version="1.0" encoding="UTF-8" ?>