[LNT] r291755 - Generate csv test report for test-suite
Kristof Beyls via llvm-commits
llvm-commits at lists.llvm.org
Thu Jan 12 01:47:21 PST 2017
Author: kbeyls
Date: Thu Jan 12 03:47:20 2017
New Revision: 291755
URL: http://llvm.org/viewvc/llvm-project?rev=291755&view=rev
Log:
Generate csv test report for test-suite
csv test reports are typically easier to quickly analyze than json or
xml reports. Therefore, also create a csv test report as part of 'lnt
runtest test-suite'; similar to how it's done for 'lnt runtest nt'.
Hopefully this is one of many steps to still take to be able to
deprecate 'lnt runtest nt'.
Differential Revision: https://reviews.llvm.org/D28562
Modified:
lnt/trunk/lnt/tests/test_suite.py
lnt/trunk/tests/runtest/test_suite.py
Modified: lnt/trunk/lnt/tests/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/tests/test_suite.py?rev=291755&r1=291754&r2=291755&view=diff
==============================================================================
--- lnt/trunk/lnt/tests/test_suite.py (original)
+++ lnt/trunk/lnt/tests/test_suite.py Thu Jan 12 03:47:20 2017
@@ -69,6 +69,25 @@ XML_REPORT_TEMPLATE = """<?xml version="
</testsuites>
"""
+CSV_REPORT_TEMPLATE = \
+"""Program;CC;CC_Time;CC_Hash;Exec;Exec_Time;Score
+{%- for suite in suites -%}
+ {%- for test in suite.tests %}
+{{ suite.name }}/{{ test.path }}/{{ test.name }};
+ {%- if test.code == "NOEXE" -%}
+ fail;*;*;
+ {%- else -%}
+ pass;{{ test.metrics.compile_time if test.metrics }};{{ test.metrics.hash if test.metrics }};
+ {%- endif -%}
+ {%- if test.code == "FAIL" or test.code == "NOEXE" -%}
+ fail;*;*;
+ {%- else -%}
+ pass;{{ test.metrics.exec_time if test.metrics }};{{ test.metrics.score if test.metrics }};
+ {%- endif -%}
+ {% endfor %}
+{%- endfor -%}
+"""
+
# _importProfile imports a single profile. It must be at the top level (and
# not within TestSuiteTest) so that multiprocessing can import it correctly.
def _importProfile(name_filename):
@@ -90,11 +109,7 @@ def _importProfile(name_filename):
str)
-def _lit_json_to_xunit_xml(json_reports):
- # type: (list) -> str
- """Take the lit report jason dicts and convert them
- to an xunit xml report for CI to digest."""
- template_engine = jinja2.Template(XML_REPORT_TEMPLATE, autoescape=True)
+def _lit_json_to_template(json_reports, template_engine):
# For now, only show first runs report.
json_report = json_reports[0]
tests_by_suite = defaultdict(list)
@@ -112,7 +127,8 @@ def _lit_json_to_xunit_xml(json_reports)
entry = {'name': test_name,
'path': '.'.join(path),
'time': time,
- 'code': code}
+ 'code': code,
+ 'metrics': tests.get('metrics', None)}
if code != "PASS":
entry['output'] = output
@@ -134,6 +150,23 @@ def _lit_json_to_xunit_xml(json_reports)
return str_template
+def _lit_json_to_xunit_xml(json_reports):
+ # type: (list) -> str
+ """Take the lit report jason dicts and convert them
+ to an xunit xml report for CI to digest."""
+ template_engine = jinja2.Template(XML_REPORT_TEMPLATE, autoescape=True)
+ return _lit_json_to_template(json_reports, template_engine)
+
+
+def _lit_json_to_csv(json_reports):
+ # type: (list) -> str
+ """Take the lit report json dicts and convert them
+ to a csv report, similar to the old test-suite make-based
+ *.report.simple.csv files."""
+ template_engine = jinja2.Template(CSV_REPORT_TEMPLATE, autoescape=True)
+ return _lit_json_to_template(json_reports, template_engine)
+
+
class TestSuiteTest(BuiltinTest):
def __init__(self):
super(TestSuiteTest, self).__init__()
@@ -477,6 +510,12 @@ class TestSuiteTest(BuiltinTest):
with open(xml_report_path, 'w') as fd:
fd.write(str_template)
+ csv_report_path = os.path.join(self._base_path,
+ 'test-results.csv')
+ str_template = _lit_json_to_csv(json_reports)
+ with open(csv_report_path, 'w') as fd:
+ fd.write(str_template)
+
return self.submit(report_path, self.opts, commit=True)
def run(self, nick, compile=True, test=True):
Modified: lnt/trunk/tests/runtest/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/test_suite.py?rev=291755&r1=291754&r2=291755&view=diff
==============================================================================
--- lnt/trunk/tests/runtest/test_suite.py (original)
+++ lnt/trunk/tests/runtest/test_suite.py Thu Jan 12 03:47:20 2017
@@ -2,7 +2,7 @@
#
# RUN: rm -rf %t.SANDBOX %t.SANDBOX2 || true
#
-# Check a basic nt run.
+# Check a basic test-suite run.
# RUN: lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX \
# RUN: --no-timestamp \
@@ -16,6 +16,7 @@
# RUN: FileCheck --check-prefix CHECK-BASIC < %t.err %s
# RUN: FileCheck --check-prefix CHECK-REPORT < %t.SANDBOX/build/report.json %s
# RUN: FileCheck --check-prefix CHECK-XML < %t.SANDBOX/build/test-results.xunit.xml %s
+# RUN: FileCheck --check-prefix CHECK-CSV < %t.SANDBOX/build/test-results.csv %s
# CHECK-REPORT: "run_order": "154331"
# CHECK-REPORT: "Name": "nts.{{[^.]+}}.compile"
@@ -44,6 +45,9 @@
# CHECK-XML: </testcase>
# CHECK-XML: </testsuite>
+# CHECK-CSV: Program;CC;CC_Time;CC_Hash;Exec;Exec_Time;Score
+# CHECK-CSV-NEXT: foo//foo;pass;1.3;xyz;pass;1.4;1.5
+
# Use the same sandbox again with --no-configure
# RUN: lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX \
More information about the llvm-commits
mailing list