[llvm] 968f58c - [lit] Include unexecuted tests in xUnit report
Julian Lettner via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 15 11:38:35 PDT 2020
Author: Julian Lettner
Date: 2020-06-15T11:36:31-07:00
New Revision: 968f58c68421bd18f09754bf11135803243a0226
URL: https://github.com/llvm/llvm-project/commit/968f58c68421bd18f09754bf11135803243a0226
DIFF: https://github.com/llvm/llvm-project/commit/968f58c68421bd18f09754bf11135803243a0226.diff
LOG: [lit] Include unexecuted tests in xUnit report
Pass in all discovered tests to report generators.
The XunitReport generator now creates testcase items for unexecuted
tests and documents why they have been skipped. This makes it easier
to compare test runs with different filters or configurations, or across
platforms.
I don't know who is using the JsonReport generator and what the
expectations there are (it doesn't have tests), so decided to preserve
the old behavior by filtering out the unexecuted tests.
Reviewed By: jdenny
Differential Revision: https://reviews.llvm.org/D81316
Added:
llvm/utils/lit/tests/Inputs/xunit-output/excluded.ini
llvm/utils/lit/tests/Inputs/xunit-output/missing_feature.ini
llvm/utils/lit/tests/Inputs/xunit-output/pass.ini
llvm/utils/lit/tests/Inputs/xunit-output/unsupported.ini
Modified:
llvm/utils/lit/lit/main.py
llvm/utils/lit/lit/reports.py
llvm/utils/lit/tests/Inputs/xunit-output/dummy_format.py
llvm/utils/lit/tests/shtest-format.py
llvm/utils/lit/tests/xunit-output.py
Removed:
################################################################################
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index b8c951360322..0c4c986d4561 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -96,17 +96,13 @@ def main(builtin_params={}):
run_tests(selected_tests, lit_config, opts, len(discovered_tests))
elapsed = time.time() - start
- # TODO(yln): eventually, all functions below should act on discovered_tests
- executed_tests = [
- t for t in selected_tests if t.result.code != lit.Test.SKIPPED]
-
if opts.time_tests:
print_histogram(discovered_tests)
print_results(discovered_tests, elapsed, opts)
for report in opts.reports:
- report.write_results(executed_tests, elapsed)
+ report.write_results(discovered_tests, elapsed)
if lit_config.numErrors:
sys.stderr.write('\n%d error(s) in tests\n' % lit_config.numErrors)
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 8499d2126be5..c23b4c714090 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -17,7 +17,8 @@ def __init__(self, output_file):
self.output_file = output_file
def write_results(self, tests, elapsed):
- assert not any(t.result.code in {lit.Test.EXCLUDED, lit.Test.SKIPPED} for t in tests)
+ unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
+ tests = [t for t in tests if t.result.code not in unexecuted_codes]
# Construct the data we will write.
data = {}
# Encode the current lit version as a schema version.
@@ -75,7 +76,6 @@ def __init__(self, output_file):
# TODO(yln): elapsed unused, put it somewhere?
def write_results(self, tests, elapsed):
- assert not any(t.result.code in {lit.Test.EXCLUDED, lit.Test.SKIPPED} for t in tests)
tests.sort(key=by_suite_and_test_path)
tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
@@ -136,4 +136,4 @@ def _get_skip_reason(self, test):
features = test.getMissingRequiredFeatures()
if features:
return 'Missing required feature(s): ' + ', '.join(features)
- return 'Skipping because of configuration'
+ return 'Unsupported configuration'
diff --git a/llvm/utils/lit/tests/Inputs/xunit-output/dummy_format.py b/llvm/utils/lit/tests/Inputs/xunit-output/dummy_format.py
index 93e48eeb8396..94410d215d39 100644
--- a/llvm/utils/lit/tests/Inputs/xunit-output/dummy_format.py
+++ b/llvm/utils/lit/tests/Inputs/xunit-output/dummy_format.py
@@ -23,6 +23,10 @@ def execute(self, test, lit_config):
result = lit.Test.Result(getattr(lit.Test, result_code),
result_output)
+ required_feature = cfg.get('global', 'required_feature', fallback=None)
+ if required_feature:
+ test.requires.append(required_feature)
+
# Load additional metrics.
for key,value_str in cfg.items('results'):
value = eval(value_str)
diff --git a/llvm/utils/lit/tests/Inputs/xunit-output/excluded.ini b/llvm/utils/lit/tests/Inputs/xunit-output/excluded.ini
new file mode 100644
index 000000000000..f62ede446872
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xunit-output/excluded.ini
@@ -0,0 +1,5 @@
+[global]
+result_code = EXCLUDED
+result_output = not shown
+
+[results]
diff --git a/llvm/utils/lit/tests/Inputs/xunit-output/missing_feature.ini b/llvm/utils/lit/tests/Inputs/xunit-output/missing_feature.ini
new file mode 100644
index 000000000000..09fb2a132929
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xunit-output/missing_feature.ini
@@ -0,0 +1,7 @@
+[global]
+result_code = UNSUPPORTED
+result_output = not shown
+
+required_feature = dummy_feature
+
+[results]
diff --git a/llvm/utils/lit/tests/Inputs/xunit-output/pass.ini b/llvm/utils/lit/tests/Inputs/xunit-output/pass.ini
new file mode 100644
index 000000000000..c1e224152639
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xunit-output/pass.ini
@@ -0,0 +1,5 @@
+[global]
+result_code = PASS
+result_output = not shown
+
+[results]
diff --git a/llvm/utils/lit/tests/Inputs/xunit-output/unsupported.ini b/llvm/utils/lit/tests/Inputs/xunit-output/unsupported.ini
new file mode 100644
index 000000000000..0d567b35a0a4
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xunit-output/unsupported.ini
@@ -0,0 +1,5 @@
+[global]
+result_code = UNSUPPORTED
+result_output = not shown
+
+[results]
diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 8cc7c679268c..2714e20e05f3 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -134,7 +134,7 @@
# XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
# XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-true.txt" time="{{[0-9]+\.[0-9]+}}">
-# XUNIT-NEXT:<skipped message="Skipping because of configuration"/>
+# XUNIT-NEXT:<skipped message="Unsupported configuration"/>
# XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-star.txt" time="{{[0-9]+\.[0-9]+}}">
# XUNIT-NEXT: <failure{{[ ]*}}>
@@ -142,7 +142,7 @@
# XUNIT-NEXT: </testcase>
# XUNIT: <testcase classname="shtest-format.unsupported_dir" name="some-test.txt" time="{{[0-9]+\.[0-9]+}}">
-# XUNIT-NEXT:<skipped message="Skipping because of configuration"/>
+# XUNIT-NEXT:<skipped message="Unsupported configuration"/>
# XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
diff --git a/llvm/utils/lit/tests/xunit-output.py b/llvm/utils/lit/tests/xunit-output.py
index 9cfe2cb9aa68..32a35f82a943 100644
--- a/llvm/utils/lit/tests/xunit-output.py
+++ b/llvm/utils/lit/tests/xunit-output.py
@@ -7,10 +7,21 @@
# RUN: sh -c 'if command -v xmllint 2>/dev/null; then xmllint --noout %t.xunit.xml; fi'
# RUN: FileCheck < %t.xunit.xml %s
-# CHECK: <?xml version="1.0" encoding="UTF-8"?>
-# CHECK: <testsuites>
-# CHECK: <testsuite name="test-data" tests="1" failures="1" skipped="0">
-# CHECK: <testcase classname="test-data.test-data" name="bad&name.ini" time="{{[0-1]}}.{{[0-9]+}}">
-# CHECK-NEXT: <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
-# CHECK: </testsuite>
-# CHECK: </testsuites>
+# CHECK: <?xml version="1.0" encoding="UTF-8"?>
+# CHECK-NEXT: <testsuites>
+# CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&name.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="excluded.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <skipped message="Test not selected (--filter, --max-tests, --run-shard)"/>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="missing_feature.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <skipped message="Missing required feature(s): dummy_feature"/>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="pass.ini" time="{{[0-1]\.[0-9]+}}"/>
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="unsupported.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT: <skipped message="Unsupported configuration"/>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: </testsuite>
+# CHECK-NEXT: </testsuites>
More information about the llvm-commits
mailing list