[LNT] r261852 - [test-suite] Pass the 'score' and 'hash' metrics along from LIT to LNT

[James Molloy via llvm-commits]

Author: jamesm
Date: Thu Feb 25 04:05:08 2016
New Revision: 261852

URL: http://llvm.org/viewvc/llvm-project?rev=261852&view=rev
Log:
[test-suite] Pass the 'score' and 'hash' metrics along from LIT to LNT

As part of this a default has been changed - 'hash' is no longer excluded by default.

The reason for this is that because we've set --exclude-stat-from-submission to 'append', it's actually impossible to un-exclude a stat that is excluded by default!

Modified:
    lnt/trunk/lnt/tests/test_suite.py
    lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results.json
    lnt/trunk/tests/runtest/test_suite.py

Modified: lnt/trunk/lnt/tests/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/tests/test_suite.py?rev=261852&r1=261851&r2=261852&view=diff
==============================================================================
--- lnt/trunk/lnt/tests/test_suite.py (original)
+++ lnt/trunk/lnt/tests/test_suite.py Thu Feb 25 04:05:08 2016
@@ -15,7 +15,7 @@ from lnt.tests.builtintest import Builti
 # test-suite/cmake/modules/DetectArchitecture.cmake. If you update this list,
 # make sure that cmake file is updated too.
 TEST_SUITE_KNOWN_ARCHITECTURES = ['ARM', 'AArch64', 'Mips', 'X86']
-KNOWN_SAMPLE_KEYS = ['compile', 'exec', 'hash']
+KNOWN_SAMPLE_KEYS = ['compile', 'exec', 'hash', 'score']
 
 
 class TestSuiteTest(BuiltinTest):
@@ -150,7 +150,7 @@ class TestSuiteTest(BuiltinTest):
                          dest="exclude_stat_from_submission",
                          help="Do not submit the stat of this type [%default]",
                          action='append', choices=KNOWN_SAMPLE_KEYS,
-                         type='choice', default=['hash'])
+                         type='choice', default=[])
         parser.add_option_group(group)
 
         group = OptionGroup(parser, "Test tools")
@@ -441,7 +441,15 @@ class TestSuiteTest(BuiltinTest):
     def _parse_lit_output(self, path, data, only_test=False):
         LIT_METRIC_TO_LNT = {
             'compile_time': 'compile',
-            'exec_time': 'exec'
+            'exec_time': 'exec',
+            'score': 'score',
+            'hash': 'hash'
+        }
+        LIT_METRIC_CONV_FN = {
+            'compile_time': float,
+            'exec_time': float,
+            'score': float,
+            'hash': str
         }
         
         # We don't use the test info, currently.
@@ -464,7 +472,8 @@ class TestSuiteTest(BuiltinTest):
                     test_samples.append(
                         lnt.testing.TestSamples(name + '.' + LIT_METRIC_TO_LNT[k],
                                                 [v],
-                                                test_info))
+                                                test_info,
+                                                LIT_METRIC_CONV_FN[k]))
 
             if self._test_failed_to_compile(raw_name, path):
                 test_samples.append(

Modified: lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results.json
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results.json?rev=261852&r1=261851&r2=261852&view=diff
==============================================================================
--- lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results.json (original)
+++ lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results.json Thu Feb 25 04:05:08 2016
@@ -4,7 +4,11 @@
             "name": "test-suite :: foo",
             "code": "PASS",
             "metrics": {
-                "compile_time": 1.3
+                "compile_time": 1.3,
+                "exec_time": 1.4,
+                "score": 1.5,
+                "hash": "xyz",
+                "unknown": "unknown"
             }
         }
     ]

Modified: lnt/trunk/tests/runtest/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/test_suite.py?rev=261852&r1=261851&r2=261852&view=diff
==============================================================================
--- lnt/trunk/tests/runtest/test_suite.py (original)
+++ lnt/trunk/tests/runtest/test_suite.py Thu Feb 25 04:05:08 2016
@@ -268,3 +268,23 @@
 # CHECK-USE-PERF: Configuring with {
 # CHECK-USE-PERF:   TEST_SUITE_USE_PERF: 'ON'
 
+# Check that hash, score, compile_time and exec_time get copied into the LNT
+# report.
+# RUN: lnt runtest test-suite \
+# RUN:     --sandbox %t.SANDBOX \
+# RUN:     --no-timestamp \
+# RUN:     --test-suite %S/Inputs/test-suite-cmake \
+# RUN:     --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
+# RUN:     --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
+# RUN:     --use-make %S/Inputs/test-suite-cmake/fake-make \
+# RUN:     --use-lit %S/Inputs/test-suite-cmake/fake-lit \
+# RUN:     --verbose \
+# RUN:     > %t.log 2> %t.err
+# RUN: FileCheck --check-prefix CHECK-METRICS < %t.SANDBOX/build/report.json %s
+# RUN: FileCheck --check-prefix CHECK-METRICS2 < %t.SANDBOX/build/report.json %s
+# CHECK-METRICS-DAG: foo.exec
+# CHECK-METRICS-DAG: foo.compile
+# CHECK-METRICS-DAG: foo.score
+# CHECK-METRICS-DAG: foo.hash
+# CHECK-METRICS2-NOT: foo.unknown
+