[test-suite] r261857 - [cmake] Add support for arbitrary metrics

[James Molloy via llvm-commits]

Author: jamesm
Date: Thu Feb 25 05:06:15 2016
New Revision: 261857

URL: http://llvm.org/viewvc/llvm-project?rev=261857&view=rev
Log:
[cmake] Add support for arbitrary metrics

This allows a .test script to specify a command to get a metric for the test. For example:

METRIC: score: grep "Score:" %o | awk '{print $2}'

Modified:
    test-suite/trunk/cmake/modules/SingleMultiSource.cmake
    test-suite/trunk/litsupport/test.py
    test-suite/trunk/litsupport/testscript.py

Modified: test-suite/trunk/cmake/modules/SingleMultiSource.cmake
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/cmake/modules/SingleMultiSource.cmake?rev=261857&r1=261856&r2=261857&view=diff
==============================================================================
--- test-suite/trunk/cmake/modules/SingleMultiSource.cmake (original)
+++ test-suite/trunk/cmake/modules/SingleMultiSource.cmake Thu Feb 25 05:06:15 2016
@@ -223,3 +223,17 @@ macro(llvm_test_verify)
     set(TESTSCRIPT "${TESTSCRIPT}VERIFY: ${JOINED_ARGUMENTS}\n")
   endif()
 endmacro()
+
+macro(llvm_test_metric)
+  CMAKE_PARSE_ARGUMENTS(ARGS "" "RUN_TYPE;METRIC" "" ${ARGN})
+  if(NOT DEFINED TESTSCRIPT)
+    set(TESTSCRIPT "" PARENT_SCOPE)
+  endif()
+  # ARGS_UNPARSED_ARGUMENTS is a semicolon-separated list. Change it into a
+  # whitespace-separated string.
+  string(REPLACE ";" " " JOINED_ARGUMENTS "${ARGS_UNPARSED_ARGUMENTS}")
+  if(NOT DEFINED ARGS_RUN_TYPE OR "${ARGS_RUN_TYPE}" STREQUAL "${TEST_SUITE_RUN_TYPE}")
+    set(TESTSCRIPT "${TESTSCRIPT}METRIC: ${ARGS_METRIC}: ${JOINED_ARGUMENTS}\n")
+  endif()
+endmacro()
+  
\ No newline at end of file

Modified: test-suite/trunk/litsupport/test.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/test.py?rev=261857&r1=261856&r2=261857&view=diff
==============================================================================
--- test-suite/trunk/litsupport/test.py (original)
+++ test-suite/trunk/litsupport/test.py Thu Feb 25 05:06:15 2016
@@ -56,7 +56,7 @@ class TestSuiteTest(FileBasedTest):
         res = testscript.parse(test.getSourcePath())
         if litConfig.noExecute:
             return lit.Test.Result(Test.PASS)
-        runscript, verifyscript = res
+        runscript, verifyscript, metricscripts = res
 
         # Apply the usual lit substitutions (%s, %S, %p, %T, ...)
         tmpDir, tmpBase = getTempPaths(test)
@@ -65,6 +65,8 @@ class TestSuiteTest(FileBasedTest):
         substitutions += [('%o', outfile)]
         runscript = applySubstitutions(runscript, substitutions)
         verifyscript = applySubstitutions(verifyscript, substitutions)
+        metricscripts = {k: applySubstitutions(v, substitutions)
+                         for k,v in metricscripts.items()}
         context = TestContext(test, litConfig, runscript, verifyscript, tmpDir,
                               tmpBase)
 
@@ -80,6 +82,7 @@ class TestSuiteTest(FileBasedTest):
         output = ""
         n_runs = 1
         runtimes = []
+        metrics = {}
         for n in range(n_runs):
             res = runScript(context, runscript)
             if isinstance(res, lit.Test.Result):
@@ -94,6 +97,15 @@ class TestSuiteTest(FileBasedTest):
                 output += "\n" + err
                 return lit.Test.Result(Test.FAIL, output)
 
+            # Execute metric extraction scripts.
+            for metric, script in metricscripts.items():
+                res = runScript(context, script)
+                if isinstance(res, lit.Test.Result):
+                    return res
+
+                out, err, exitCode, timeoutInfo = res
+                metrics.setdefault(metric, list()).append(float(out))
+            
             try:
                 runtime = runsafely.getTime(context)
                 runtimes.append(runtime)
@@ -128,6 +140,8 @@ class TestSuiteTest(FileBasedTest):
         result = lit.Test.Result(Test.PASS, output)
         if len(runtimes) > 0:
             result.addMetric('exec_time', lit.Test.toMetricValue(runtimes[0]))
+        for metric, values in metrics.items():
+            result.addMetric(metric, lit.Test.toMetricValue(values[0]))
         compiletime.collect(context, result)
 
         return result

Modified: test-suite/trunk/litsupport/testscript.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/testscript.py?rev=261857&r1=261856&r2=261857&view=diff
==============================================================================
--- test-suite/trunk/litsupport/testscript.py (original)
+++ test-suite/trunk/litsupport/testscript.py Thu Feb 25 05:06:15 2016
@@ -22,13 +22,18 @@ def parse(filename):
     # Collect the test lines from the script.
     runscript = []
     verifyscript = []
-    keywords = ['RUN:', 'VERIFY:']
+    metricscripts = {}
+    keywords = ['RUN:', 'VERIFY:', 'METRIC:']
     for line_number, command_type, ln in \
             parseIntegratedTestScriptCommands(filename, keywords):
         if command_type == 'RUN':
             _parseShellCommand(runscript, ln)
         elif command_type == 'VERIFY':
             _parseShellCommand(verifyscript, ln)
+        elif command_type == 'METRIC':
+            metric, ln = ln.split(':', 1)
+            metricscript = metricscripts.setdefault(metric.strip(), list())
+            _parseShellCommand(metricscript, ln)
         else:
             raise ValueError("unknown script command type: %r" % (
                              command_type,))
@@ -43,4 +48,4 @@ def parse(filename):
             raise ValueError("Test has unterminated RUN/VERIFY lines " +
                              "(ending with '\\')")
 
-    return runscript, verifyscript
+    return runscript, verifyscript, metricscripts