[llvm] [lit] Add --report-failures-only option for lit test reports (PR #115439)
Rakshit Patel via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 8 09:54:24 PST 2024
https://github.com/rpatel321 updated https://github.com/llvm/llvm-project/pull/115439
>From 17f4900a17b24bf288727c9f70764ad08aa18170 Mon Sep 17 00:00:00 2001
From: Rakshit Patel <Rakshit.Patel at sony.com>
Date: Thu, 7 Nov 2024 23:49:20 -0800
Subject: [PATCH 1/3] Add --report-failures-only option for lit test reports
Co-authored-by: Greg Bedwell <greg.bedwell at sony.com>
---
llvm/utils/lit/lit/cl_arguments.py | 6 ++++++
llvm/utils/lit/lit/main.py | 4 ++++
llvm/utils/lit/lit/reports.py | 6 ++----
.../tests/xunit-output-report-failures-only.py | 17 +++++++++++++++++
4 files changed, 29 insertions(+), 4 deletions(-)
create mode 100644 llvm/utils/lit/tests/xunit-output-report-failures-only.py
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 3e5488f388ccfa..4ca0e72216ed58 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -165,6 +165,12 @@ def parse_args():
type=lit.reports.XunitReport,
help="Write XUnit-compatible XML test reports to the specified file",
)
+ execution_group.add_argument(
+ "--report-failures-only",
+ help="When writing a test report, do not include results for "
+ "tests that completed successfully or were not run",
+ action="store_true"
+ )
execution_group.add_argument(
"--resultdb-output",
type=lit.reports.ResultDBReport,
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 24ba804f0c363f..e8423ec3163a97 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -137,6 +137,10 @@ def main(builtin_params={}):
print_results(discovered_tests, elapsed, opts)
tests_for_report = selected_tests if opts.shard else discovered_tests
+ if opts.report_failures_only:
+ # Only report tests that failed
+ tests_for_report = [t for t in tests_for_report if t.isFailure()]
+
for report in opts.reports:
report.write_results(tests_for_report, elapsed)
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 8312dcddc769ae..845aeda7b47f22 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -22,6 +22,8 @@ def __init__(self, output_file):
self.output_file = output_file
# Set by the option parser later.
self.use_unique_output_file_name = False
+ self.skipped_codes = {lit.Test.EXCLUDED,
+ lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
def write_results(self, tests, elapsed):
if self.use_unique_output_file_name:
@@ -114,8 +116,6 @@ def remove_invalid_xml_chars(s):
class XunitReport(Report):
- skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
-
def _write_results_to_file(self, tests, elapsed, file):
tests.sort(key=by_suite_and_test_path)
tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
@@ -273,8 +273,6 @@ def _write_results_to_file(self, tests, elapsed, file):
class TimeTraceReport(Report):
- skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
-
def _write_results_to_file(self, tests, elapsed, file):
# Find when first test started so we can make start times relative.
first_start_time = min([t.result.start for t in tests])
diff --git a/llvm/utils/lit/tests/xunit-output-report-failures-only.py b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
new file mode 100644
index 00000000000000..3c8c49362aa175
--- /dev/null
+++ b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
@@ -0,0 +1,17 @@
+# UNSUPPORTED: system-windows
+
+# Check xunit output
+# RUN: rm -rf %t.xunit.xml
+# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# If xmllint is installed verify that the generated xml is well-formed
+# RUN: sh -c 'if command -v xmllint 2>/dev/null; then xmllint --noout %t.xunit.xml; fi'
+# RUN: FileCheck < %t.xunit.xml %s
+
+# CHECK: <?xml version="1.0" encoding="UTF-8"?>
+# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
+# CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3" time="{{[0-9.]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&name.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: </testsuite>
+# CHECK-NEXT: </testsuites>
>From 4140e14bffc57f41c9009f97a55e34ba79343c0f Mon Sep 17 00:00:00 2001
From: Rakshit Patel <rakshit.patel at sony.com>
Date: Fri, 8 Nov 2024 09:47:42 -0800
Subject: [PATCH 2/3] Add full stop to comment
Co-authored-by: James Henderson <James.Henderson at sony.com>
---
llvm/utils/lit/lit/main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index e8423ec3163a97..ba80330d224005 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -138,7 +138,7 @@ def main(builtin_params={}):
tests_for_report = selected_tests if opts.shard else discovered_tests
if opts.report_failures_only:
- # Only report tests that failed
+ # Only report tests that failed.
tests_for_report = [t for t in tests_for_report if t.isFailure()]
for report in opts.reports:
>From edc093502b8db94b78dd61621c524803e4e7c226 Mon Sep 17 00:00:00 2001
From: Rakshit Patel <rakshit.patel at sony.com>
Date: Fri, 8 Nov 2024 09:54:16 -0800
Subject: [PATCH 3/3] Use double comment markers and full stop
Co-authored-by: James Henderson <James.Henderson at sony.com>
---
llvm/utils/lit/tests/xunit-output-report-failures-only.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/tests/xunit-output-report-failures-only.py b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
index 3c8c49362aa175..3623b99951f18c 100644
--- a/llvm/utils/lit/tests/xunit-output-report-failures-only.py
+++ b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
@@ -1,6 +1,6 @@
# UNSUPPORTED: system-windows
-# Check xunit output
+## Check xunit output.
# RUN: rm -rf %t.xunit.xml
# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
# If xmllint is installed verify that the generated xml is well-formed
More information about the llvm-commits
mailing list