[PATCH] D84235: [lit][xunit output] Don't include tests skipped due to sharding

[Alexander Richardson via Phabricator via llvm-commits]

arichardson created this revision.
arichardson added a reviewer: yln.
Herald added subscribers: llvm-commits, delcypher.
Herald added a project: LLVM.

When running multiple shards, don't include skipped tests in the xunit
output since merging the files will result in duplicates.
In our CHERI Jenkins CI, I configured the libc++ tests to run using sharding
(since we are testing using a single-CPU QEMU). We then merge the generated
XUnit xml files to produce a final result, but if the individual XMLs
report tests excluded due to sharding each test is included N times in the
final result. This also makes it difficult to find the tests that were
skipped due to missing REQUIRES: etc.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D84235

Files:
  llvm/utils/lit/lit/main.py
  llvm/utils/lit/lit/reports.py


Index: llvm/utils/lit/lit/reports.py
===================================================================
--- llvm/utils/lit/lit/reports.py
+++ llvm/utils/lit/lit/reports.py
@@ -16,7 +16,7 @@
     def __init__(self, output_file):
         self.output_file = output_file
 
-    def write_results(self, tests, elapsed):
+    def write_results(self, tests, elapsed, shard):
         unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
         tests = [t for t in tests if t.result.code not in unexecuted_codes]
         # Construct the data we will write.
@@ -86,7 +86,7 @@
         self.skipped_codes = {lit.Test.EXCLUDED,
                               lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
-    def write_results(self, tests, elapsed):
+    def write_results(self, tests, elapsed, shard):
         tests.sort(key=by_suite_and_test_path)
         tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
 
@@ -94,10 +94,15 @@
             file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
             file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
             for suite, test_iter in tests_by_suite:
-                self._write_testsuite(file, suite, list(test_iter))
+                self._write_testsuite(file, suite, list(test_iter), shard)
             file.write('</testsuites>\n')
 
-    def _write_testsuite(self, file, suite, tests):
+    def _write_testsuite(self, file, suite, tests, shard):
+        print("_write_testsuite:", shard)
+        # When running multiple shards, don't include skipped tests in the xunit output
+        # since merging the files will result in duplicates otherwise:
+        if shard is not None:
+            tests = [t for t in tests if t.result.code != lit.Test.EXCLUDED]
         skipped = sum(1 for t in tests if t.result.code in self.skipped_codes)
         failures = sum(1 for t in tests if t.isFailure())
 
@@ -144,7 +149,7 @@
     def _get_skip_reason(self, test):
         code = test.result.code
         if code == lit.Test.EXCLUDED:
-            return 'Test not selected (--filter, --max-tests, --run-shard)'
+            return 'Test not selected (--filter, --max-tests)'
         if code == lit.Test.SKIPPED:
             return 'User interrupt'
 
Index: llvm/utils/lit/lit/main.py
===================================================================
--- llvm/utils/lit/lit/main.py
+++ llvm/utils/lit/lit/main.py
@@ -102,7 +102,7 @@
     print_results(discovered_tests, elapsed, opts)
 
     for report in opts.reports:
-        report.write_results(discovered_tests, elapsed)
+        report.write_results(discovered_tests, elapsed, opts.runShard)
 
     if lit_config.numErrors:
         sys.stderr.write('\n%d error(s) in tests\n' % lit_config.numErrors)


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D84235.279495.patch
Type: text/x-patch
Size: 2755 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20200721/85ebb82c/attachment.bin>