[llvm] [lit] Add --report-failures-only option for lit test reports (PR #115439)
Rakshit Patel via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 12 10:54:03 PST 2024
https://github.com/rpatel321 updated https://github.com/llvm/llvm-project/pull/115439
>From 17f4900a17b24bf288727c9f70764ad08aa18170 Mon Sep 17 00:00:00 2001
From: Rakshit Patel <Rakshit.Patel at sony.com>
Date: Thu, 7 Nov 2024 23:49:20 -0800
Subject: [PATCH 1/8] Add --report-failures-only option for lit test reports
Co-authored-by: Greg Bedwell <greg.bedwell at sony.com>
---
llvm/utils/lit/lit/cl_arguments.py | 6 ++++++
llvm/utils/lit/lit/main.py | 4 ++++
llvm/utils/lit/lit/reports.py | 6 ++----
.../tests/xunit-output-report-failures-only.py | 17 +++++++++++++++++
4 files changed, 29 insertions(+), 4 deletions(-)
create mode 100644 llvm/utils/lit/tests/xunit-output-report-failures-only.py
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 3e5488f388ccfa..4ca0e72216ed58 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -165,6 +165,12 @@ def parse_args():
type=lit.reports.XunitReport,
help="Write XUnit-compatible XML test reports to the specified file",
)
+ execution_group.add_argument(
+ "--report-failures-only",
+ help="When writing a test report, do not include results for "
+ "tests that completed successfully or were not run",
+ action="store_true"
+ )
execution_group.add_argument(
"--resultdb-output",
type=lit.reports.ResultDBReport,
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 24ba804f0c363f..e8423ec3163a97 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -137,6 +137,10 @@ def main(builtin_params={}):
print_results(discovered_tests, elapsed, opts)
tests_for_report = selected_tests if opts.shard else discovered_tests
+ if opts.report_failures_only:
+ # Only report tests that failed
+ tests_for_report = [t for t in tests_for_report if t.isFailure()]
+
for report in opts.reports:
report.write_results(tests_for_report, elapsed)
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 8312dcddc769ae..845aeda7b47f22 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -22,6 +22,8 @@ def __init__(self, output_file):
self.output_file = output_file
# Set by the option parser later.
self.use_unique_output_file_name = False
+ self.skipped_codes = {lit.Test.EXCLUDED,
+ lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
def write_results(self, tests, elapsed):
if self.use_unique_output_file_name:
@@ -114,8 +116,6 @@ def remove_invalid_xml_chars(s):
class XunitReport(Report):
- skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
-
def _write_results_to_file(self, tests, elapsed, file):
tests.sort(key=by_suite_and_test_path)
tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
@@ -273,8 +273,6 @@ def _write_results_to_file(self, tests, elapsed, file):
class TimeTraceReport(Report):
- skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
-
def _write_results_to_file(self, tests, elapsed, file):
# Find when first test started so we can make start times relative.
first_start_time = min([t.result.start for t in tests])
diff --git a/llvm/utils/lit/tests/xunit-output-report-failures-only.py b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
new file mode 100644
index 00000000000000..3c8c49362aa175
--- /dev/null
+++ b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
@@ -0,0 +1,17 @@
+# UNSUPPORTED: system-windows
+
+# Check xunit output
+# RUN: rm -rf %t.xunit.xml
+# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# If xmllint is installed verify that the generated xml is well-formed
+# RUN: sh -c 'if command -v xmllint 2>/dev/null; then xmllint --noout %t.xunit.xml; fi'
+# RUN: FileCheck < %t.xunit.xml %s
+
+# CHECK: <?xml version="1.0" encoding="UTF-8"?>
+# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
+# CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3" time="{{[0-9.]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&name.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: </testsuite>
+# CHECK-NEXT: </testsuites>
>From 4140e14bffc57f41c9009f97a55e34ba79343c0f Mon Sep 17 00:00:00 2001
From: Rakshit Patel <rakshit.patel at sony.com>
Date: Fri, 8 Nov 2024 09:47:42 -0800
Subject: [PATCH 2/8] Add full stop to comment
Co-authored-by: James Henderson <James.Henderson at sony.com>
---
llvm/utils/lit/lit/main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index e8423ec3163a97..ba80330d224005 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -138,7 +138,7 @@ def main(builtin_params={}):
tests_for_report = selected_tests if opts.shard else discovered_tests
if opts.report_failures_only:
- # Only report tests that failed
+ # Only report tests that failed.
tests_for_report = [t for t in tests_for_report if t.isFailure()]
for report in opts.reports:
>From edc093502b8db94b78dd61621c524803e4e7c226 Mon Sep 17 00:00:00 2001
From: Rakshit Patel <rakshit.patel at sony.com>
Date: Fri, 8 Nov 2024 09:54:16 -0800
Subject: [PATCH 3/8] Use double comment markers and full stop
Co-authored-by: James Henderson <James.Henderson at sony.com>
---
llvm/utils/lit/tests/xunit-output-report-failures-only.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/tests/xunit-output-report-failures-only.py b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
index 3c8c49362aa175..3623b99951f18c 100644
--- a/llvm/utils/lit/tests/xunit-output-report-failures-only.py
+++ b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
@@ -1,6 +1,6 @@
# UNSUPPORTED: system-windows
-# Check xunit output
+## Check xunit output.
# RUN: rm -rf %t.xunit.xml
# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
# If xmllint is installed verify that the generated xml is well-formed
>From 1ebd26cb95b23a01f0d12e24e962ab1a2736f6c3 Mon Sep 17 00:00:00 2001
From: Rakshit Patel <Rakshit.Patel at sony.com>
Date: Fri, 8 Nov 2024 09:56:03 -0800
Subject: [PATCH 4/8] Move skipped codes back to XunitReport and
TimeTraceReport
---
llvm/utils/lit/lit/reports.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 845aeda7b47f22..8312dcddc769ae 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -22,8 +22,6 @@ def __init__(self, output_file):
self.output_file = output_file
# Set by the option parser later.
self.use_unique_output_file_name = False
- self.skipped_codes = {lit.Test.EXCLUDED,
- lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
def write_results(self, tests, elapsed):
if self.use_unique_output_file_name:
@@ -116,6 +114,8 @@ def remove_invalid_xml_chars(s):
class XunitReport(Report):
+ skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
+
def _write_results_to_file(self, tests, elapsed, file):
tests.sort(key=by_suite_and_test_path)
tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
@@ -273,6 +273,8 @@ def _write_results_to_file(self, tests, elapsed, file):
class TimeTraceReport(Report):
+ skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
+
def _write_results_to_file(self, tests, elapsed, file):
# Find when first test started so we can make start times relative.
first_start_time = min([t.result.start for t in tests])
>From 5d16c986e374a69da9c7859c39b3c03b18b72b3a Mon Sep 17 00:00:00 2001
From: Rakshit Patel <Rakshit.Patel at sony.com>
Date: Fri, 8 Nov 2024 10:21:00 -0800
Subject: [PATCH 5/8] Remove redundant xmllint check
---
llvm/utils/lit/tests/xunit-output-report-failures-only.py | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/llvm/utils/lit/tests/xunit-output-report-failures-only.py b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
index 3623b99951f18c..8f8bf9c0837e91 100644
--- a/llvm/utils/lit/tests/xunit-output-report-failures-only.py
+++ b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
@@ -1,15 +1,11 @@
-# UNSUPPORTED: system-windows
-
## Check xunit output.
# RUN: rm -rf %t.xunit.xml
# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
-# If xmllint is installed verify that the generated xml is well-formed
-# RUN: sh -c 'if command -v xmllint 2>/dev/null; then xmllint --noout %t.xunit.xml; fi'
-# RUN: FileCheck < %t.xunit.xml %s
+# RUN: FileCheck --input-file=%t.xunit.xml %s
# CHECK: <?xml version="1.0" encoding="UTF-8"?>
# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
-# CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3" time="{{[0-9.]+}}">
+# CHECK-NEXT: <testsuite name="test-data" tests="1" failures="1" skipped="0" time="{{[0-9.]+}}">
# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&name.ini" time="{{[0-1]\.[0-9]+}}">
# CHECK-NEXT: <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
# CHECK-NEXT: </testcase>
>From e09e4f1ecc3239c1f170838c887c3be14b0b7584 Mon Sep 17 00:00:00 2001
From: Rakshit Patel <Rakshit.Patel at sony.com>
Date: Fri, 8 Nov 2024 10:30:18 -0800
Subject: [PATCH 6/8] Remove rm -rf command
---
llvm/utils/lit/tests/xunit-output-report-failures-only.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/utils/lit/tests/xunit-output-report-failures-only.py b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
index 8f8bf9c0837e91..e15fd6a009f99a 100644
--- a/llvm/utils/lit/tests/xunit-output-report-failures-only.py
+++ b/llvm/utils/lit/tests/xunit-output-report-failures-only.py
@@ -1,5 +1,4 @@
## Check xunit output.
-# RUN: rm -rf %t.xunit.xml
# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
# RUN: FileCheck --input-file=%t.xunit.xml %s
>From 784e24e239cde592a2b30f06a947f0e4ca995efd Mon Sep 17 00:00:00 2001
From: Rakshit Patel <Rakshit.Patel at sony.com>
Date: Fri, 8 Nov 2024 21:48:37 -0800
Subject: [PATCH 7/8] Update documentation
---
llvm/docs/CommandGuide/lit.rst | 4 ++++
llvm/utils/lit/lit/cl_arguments.py | 4 ++--
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index af8a1a08be535d..8c0e275e1f8ca3 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -191,6 +191,10 @@ EXECUTION OPTIONS
Write XUnit-compatible XML test reports to the specified file.
+.. option:: --report-failures-only
+
+ Only include unresolved, timed out, failed and unexpectedly passed tests in the report.
+
.. option:: --resultdb-output RESULTDB_OUTPUT
Write LuCI ResultDB compatible JSON to the specified file.
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 4ca0e72216ed58..814c3cb367b42c 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -167,8 +167,8 @@ def parse_args():
)
execution_group.add_argument(
"--report-failures-only",
- help="When writing a test report, do not include results for "
- "tests that completed successfully or were not run",
+ help="Only include unresolved, timed out, failed"
+ " and unexpectedly passed tests in the report",
action="store_true"
)
execution_group.add_argument(
>From 6793a94be175c343b252f3d641de844f39389772 Mon Sep 17 00:00:00 2001
From: Rakshit Patel <Rakshit.Patel at sony.com>
Date: Tue, 12 Nov 2024 10:53:07 -0800
Subject: [PATCH 8/8] Fix formatting
---
llvm/utils/lit/lit/cl_arguments.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 814c3cb367b42c..d9bbc3e0b987a9 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -169,7 +169,7 @@ def parse_args():
"--report-failures-only",
help="Only include unresolved, timed out, failed"
" and unexpectedly passed tests in the report",
- action="store_true"
+ action="store_true",
)
execution_group.add_argument(
"--resultdb-output",
More information about the llvm-commits
mailing list