[LNT] r291755 - Generate csv test report for test-suite

[Kristof Beyls via llvm-commits]

Author: kbeyls
Date: Thu Jan 12 03:47:20 2017
New Revision: 291755

URL: http://llvm.org/viewvc/llvm-project?rev=291755&view=rev
Log:
Generate csv test report for test-suite

csv test reports are typically easier to quickly analyze than json or
xml reports. Therefore, also create a csv test report as part of 'lnt
runtest test-suite'; similar to how it's done for 'lnt runtest nt'.

Hopefully this is one of many steps to still take to be able to
deprecate 'lnt runtest nt'.

Differential Revision: https://reviews.llvm.org/D28562

Modified:
    lnt/trunk/lnt/tests/test_suite.py
    lnt/trunk/tests/runtest/test_suite.py

Modified: lnt/trunk/lnt/tests/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/tests/test_suite.py?rev=291755&r1=291754&r2=291755&view=diff
==============================================================================
--- lnt/trunk/lnt/tests/test_suite.py (original)
+++ lnt/trunk/lnt/tests/test_suite.py Thu Jan 12 03:47:20 2017
@@ -69,6 +69,25 @@ XML_REPORT_TEMPLATE = """<?xml version="
 </testsuites>
 """
 
+CSV_REPORT_TEMPLATE = \
+"""Program;CC;CC_Time;CC_Hash;Exec;Exec_Time;Score
+{%- for suite in suites -%}
+    {%- for test in suite.tests %}
+{{ suite.name }}/{{ test.path }}/{{ test.name }};
+        {%- if test.code == "NOEXE" -%}
+            fail;*;*;
+        {%- else -%}
+            pass;{{ test.metrics.compile_time if test.metrics }};{{ test.metrics.hash if test.metrics }};
+        {%- endif -%}
+        {%- if test.code == "FAIL" or test.code == "NOEXE" -%}
+            fail;*;*;
+        {%- else -%}
+            pass;{{ test.metrics.exec_time if test.metrics }};{{ test.metrics.score if test.metrics }};
+        {%- endif -%}
+    {% endfor %}
+{%- endfor -%}
+"""
+
 # _importProfile imports a single profile. It must be at the top level (and
 # not within TestSuiteTest) so that multiprocessing can import it correctly.
 def _importProfile(name_filename):
@@ -90,11 +109,7 @@ def _importProfile(name_filename):
                                    str)
 
 
-def _lit_json_to_xunit_xml(json_reports):
-    # type: (list) -> str
-    """Take the lit report jason dicts and convert them
-    to an xunit xml report for CI to digest."""
-    template_engine = jinja2.Template(XML_REPORT_TEMPLATE, autoescape=True)
+def _lit_json_to_template(json_reports, template_engine):
     # For now, only show first runs report.
     json_report = json_reports[0]
     tests_by_suite = defaultdict(list)
@@ -112,7 +127,8 @@ def _lit_json_to_xunit_xml(json_reports)
         entry = {'name': test_name,
                  'path': '.'.join(path),
                  'time': time,
-                 'code': code}
+                 'code': code,
+                 'metrics': tests.get('metrics', None)}
         if code != "PASS":
             entry['output'] = output
 
@@ -134,6 +150,23 @@ def _lit_json_to_xunit_xml(json_reports)
     return str_template
 
 
+def _lit_json_to_xunit_xml(json_reports):
+    # type: (list) -> str
+    """Take the lit report jason dicts and convert them
+    to an xunit xml report for CI to digest."""
+    template_engine = jinja2.Template(XML_REPORT_TEMPLATE, autoescape=True)
+    return _lit_json_to_template(json_reports, template_engine)
+
+
+def _lit_json_to_csv(json_reports):
+    # type: (list) -> str
+    """Take the lit report json dicts and convert them
+    to a csv report, similar to the old test-suite make-based
+    *.report.simple.csv files."""
+    template_engine = jinja2.Template(CSV_REPORT_TEMPLATE, autoescape=True)
+    return _lit_json_to_template(json_reports, template_engine)
+
+
 class TestSuiteTest(BuiltinTest):
     def __init__(self):
         super(TestSuiteTest, self).__init__()
@@ -477,6 +510,12 @@ class TestSuiteTest(BuiltinTest):
         with open(xml_report_path, 'w') as fd:
             fd.write(str_template)
 
+        csv_report_path = os.path.join(self._base_path,
+                                       'test-results.csv')
+        str_template = _lit_json_to_csv(json_reports)
+        with open(csv_report_path, 'w') as fd:
+            fd.write(str_template)
+
         return self.submit(report_path, self.opts, commit=True)
 
     def run(self, nick, compile=True, test=True):

Modified: lnt/trunk/tests/runtest/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/test_suite.py?rev=291755&r1=291754&r2=291755&view=diff
==============================================================================
--- lnt/trunk/tests/runtest/test_suite.py (original)
+++ lnt/trunk/tests/runtest/test_suite.py Thu Jan 12 03:47:20 2017
@@ -2,7 +2,7 @@
 #
 # RUN: rm -rf  %t.SANDBOX  %t.SANDBOX2 || true
 #
-# Check a basic nt run.
+# Check a basic test-suite run.
 # RUN: lnt runtest test-suite \
 # RUN:     --sandbox %t.SANDBOX \
 # RUN:     --no-timestamp \
@@ -16,6 +16,7 @@
 # RUN: FileCheck  --check-prefix CHECK-BASIC < %t.err %s
 # RUN: FileCheck  --check-prefix CHECK-REPORT < %t.SANDBOX/build/report.json %s
 # RUN: FileCheck  --check-prefix CHECK-XML < %t.SANDBOX/build/test-results.xunit.xml %s
+# RUN: FileCheck  --check-prefix CHECK-CSV < %t.SANDBOX/build/test-results.csv %s
 
 # CHECK-REPORT: "run_order": "154331"
 # CHECK-REPORT: "Name": "nts.{{[^.]+}}.compile"
@@ -44,6 +45,9 @@
 # CHECK-XML:     </testcase>
 # CHECK-XML: </testsuite>
 
+# CHECK-CSV: Program;CC;CC_Time;CC_Hash;Exec;Exec_Time;Score
+# CHECK-CSV-NEXT: foo//foo;pass;1.3;xyz;pass;1.4;1.5
+
 # Use the same sandbox again with --no-configure
 # RUN: lnt runtest test-suite \
 # RUN:     --sandbox %t.SANDBOX \