[llvm] [lit] Add --report-failures-only option for lit test reports (PR #115439)

[Rakshit Patel via llvm-commits]

https://github.com/rpatel321 updated https://github.com/llvm/llvm-project/pull/115439

>From 17f4900a17b24bf288727c9f70764ad08aa18170 Mon Sep 17 00:00:00 2001
From: Rakshit Patel <Rakshit.Patel at sony.com>
Date: Thu, 7 Nov 2024 23:49:20 -0800
Subject: [PATCH 1/2] Add --report-failures-only option for lit test reports

Co-authored-by: Greg Bedwell <greg.bedwell at sony.com>
---
 llvm/utils/lit/lit/cl_arguments.py              |  6 ++++++
 llvm/utils/lit/lit/main.py                      |  4 ++++
 llvm/utils/lit/lit/reports.py                   |  6 ++----
 .../tests/xunit-output-report-failures-only.py  | 17 +++++++++++++++++
 4 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 llvm/utils/lit/tests/xunit-output-report-failures-only.py

diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 3e5488f388ccfa..4ca0e72216ed58 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -165,6 +165,12 @@ def parse_args():
         type=lit.reports.XunitReport,
         help="Write XUnit-compatible XML test reports to the specified file",
     )
+    execution_group.add_argument(
+        "--report-failures-only",
+        help="When writing a test report, do not include results for "
+                 "tests that completed successfully or were not run",
+        action="store_true"
+    )
     execution_group.add_argument(
         "--resultdb-output",
         type=lit.reports.ResultDBReport,
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 24ba804f0c363f..e8423ec3163a97 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -137,6 +137,10 @@ def main(builtin_params={}):
     print_results(discovered_tests, elapsed, opts)
 
     tests_for_report = selected_tests if opts.shard else discovered_tests
+    if opts.report_failures_only:
+        # Only report tests that failed
+        tests_for_report = [t for t in tests_for_report if t.isFailure()]
+
     for report in opts.reports:
         report.write_results(tests_for_report, elapsed)
 
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 8312dcddc769ae..845aeda7b47f22 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -22,6 +22,8 @@ def __init__(self, output_file):
         self.output_file = output_file
         # Set by the option parser later.
         self.use_unique_output_file_name = False
+        self.skipped_codes = {lit.Test.EXCLUDED,
+                              lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
     def write_results(self, tests, elapsed):
         if self.use_unique_output_file_name:
@@ -114,8 +116,6 @@ def remove_invalid_xml_chars(s):
 
 
 class XunitReport(Report):
-    skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
-
     def _write_results_to_file(self, tests, elapsed, file):
         tests.sort(key=by_suite_and_test_path)
         tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
@@ -273,8 +273,6 @@ def _write_results_to_file(self, tests, elapsed, file):
 
 
 class TimeTraceReport(Report):
-    skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
-
     def _write_results_to_file(self, tests, elapsed, file):
         # Find when first test started so we can make start times relative.
         first_start_time = min([t.result.start for t in tests])
diff --git a/llvm/utils/lit/tests/xunit-output-report-failures-only.py b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
new file mode 100644
index 00000000000000..3c8c49362aa175
--- /dev/null
+++ b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
@@ -0,0 +1,17 @@
+# UNSUPPORTED: system-windows
+
+# Check xunit output
+# RUN: rm -rf %t.xunit.xml
+# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# If xmllint is installed verify that the generated xml is well-formed
+# RUN: sh -c 'if command -v xmllint 2>/dev/null; then xmllint --noout %t.xunit.xml; fi'
+# RUN: FileCheck < %t.xunit.xml %s
+
+# CHECK:      <?xml version="1.0" encoding="UTF-8"?>
+# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
+# CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3" time="{{[0-9.]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&name.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT:   <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: </testsuite>
+# CHECK-NEXT: </testsuites>

>From 4140e14bffc57f41c9009f97a55e34ba79343c0f Mon Sep 17 00:00:00 2001
From: Rakshit Patel <rakshit.patel at sony.com>
Date: Fri, 8 Nov 2024 09:47:42 -0800
Subject: [PATCH 2/2] Add full stop to comment

Co-authored-by: James Henderson <James.Henderson at sony.com>
---
 llvm/utils/lit/lit/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index e8423ec3163a97..ba80330d224005 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -138,7 +138,7 @@ def main(builtin_params={}):
 
     tests_for_report = selected_tests if opts.shard else discovered_tests
     if opts.report_failures_only:
-        # Only report tests that failed
+        # Only report tests that failed.
         tests_for_report = [t for t in tests_for_report if t.isFailure()]
 
     for report in opts.reports: