[llvm] c63e83f - [lit] Add --report-failures-only option for lit test reports (#115439)

[via llvm-commits]

Author: Rakshit Patel
Date: 2024-11-13T08:30:33Z
New Revision: c63e83f49575c024cf89fce9bc95d64988f3177b

URL: https://github.com/llvm/llvm-project/commit/c63e83f49575c024cf89fce9bc95d64988f3177b
DIFF: https://github.com/llvm/llvm-project/commit/c63e83f49575c024cf89fce9bc95d64988f3177b.diff

LOG: [lit] Add --report-failures-only option for lit test reports (#115439)

- Add option (--report-failures-only) to generate a reduced report for
lit tests that only includes failing tests
- This is a continuation of proposed patches by @gregbedwell here:
    - https://reviews.llvm.org/D143516
    - https://reviews.llvm.org/D143519

---------

Co-authored-by: Greg Bedwell <greg.bedwell at sony.com>
Co-authored-by: James Henderson <James.Henderson at sony.com>

Added: 
    llvm/utils/lit/tests/xunit-output-report-failures-only.py

Modified: 
    llvm/docs/CommandGuide/lit.rst
    llvm/utils/lit/lit/cl_arguments.py
    llvm/utils/lit/lit/main.py

Removed: 
    


################################################################################
diff  --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index af8a1a08be535d..8c0e275e1f8ca3 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -191,6 +191,10 @@ EXECUTION OPTIONS
 
  Write XUnit-compatible XML test reports to the specified file.
 
+.. option:: --report-failures-only
+
+ Only include unresolved, timed out, failed and unexpectedly passed tests in the report.
+
 .. option:: --resultdb-output RESULTDB_OUTPUT
 
  Write LuCI ResultDB compatible JSON to the specified file.

diff  --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index a401c9a09e081b..1d776e0216a1e5 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -165,6 +165,12 @@ def parse_args():
         type=lit.reports.XunitReport,
         help="Write XUnit-compatible XML test reports to the specified file",
     )
+    execution_group.add_argument(
+        "--report-failures-only",
+        help="Only include unresolved, timed out, failed"
+        " and unexpectedly passed tests in the report",
+        action="store_true",
+    )
     execution_group.add_argument(
         "--resultdb-output",
         type=lit.reports.ResultDBReport,

diff  --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 24ba804f0c363f..ba80330d224005 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -137,6 +137,10 @@ def main(builtin_params={}):
     print_results(discovered_tests, elapsed, opts)
 
     tests_for_report = selected_tests if opts.shard else discovered_tests
+    if opts.report_failures_only:
+        # Only report tests that failed.
+        tests_for_report = [t for t in tests_for_report if t.isFailure()]
+
     for report in opts.reports:
         report.write_results(tests_for_report, elapsed)
 

diff  --git a/llvm/utils/lit/tests/xunit-output-report-failures-only.py b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
new file mode 100644
index 00000000000000..e15fd6a009f99a
--- /dev/null
+++ b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
@@ -0,0 +1,12 @@
+## Check xunit output.
+# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# RUN: FileCheck --input-file=%t.xunit.xml %s
+
+# CHECK:      <?xml version="1.0" encoding="UTF-8"?>
+# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
+# CHECK-NEXT: <testsuite name="test-data" tests="1" failures="1" skipped="0" time="{{[0-9.]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&name.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT:   <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: </testsuite>
+# CHECK-NEXT: </testsuites>