[test-suite] r261857 - [cmake] Add support for arbitrary metrics
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 25 13:33:33 PST 2016
Hi James,
thanks for working on the test-suite. It's nice to see new capabilities added to the lit system.
Are you planing to add tests that use this? If not we should really have at least an example/unit-test type thing in the repository.
- Matthias
> On Feb 25, 2016, at 3:06 AM, James Molloy via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>
> Author: jamesm
> Date: Thu Feb 25 05:06:15 2016
> New Revision: 261857
>
> URL: http://llvm.org/viewvc/llvm-project?rev=261857&view=rev
> Log:
> [cmake] Add support for arbitrary metrics
>
> This allows a .test script to specify a command to get a metric for the test. For example:
>
> METRIC: score: grep "Score:" %o | awk '{print $2}'
>
> Modified:
> test-suite/trunk/cmake/modules/SingleMultiSource.cmake
> test-suite/trunk/litsupport/test.py
> test-suite/trunk/litsupport/testscript.py
>
> Modified: test-suite/trunk/cmake/modules/SingleMultiSource.cmake
> URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/cmake/modules/SingleMultiSource.cmake?rev=261857&r1=261856&r2=261857&view=diff
> ==============================================================================
> --- test-suite/trunk/cmake/modules/SingleMultiSource.cmake (original)
> +++ test-suite/trunk/cmake/modules/SingleMultiSource.cmake Thu Feb 25 05:06:15 2016
> @@ -223,3 +223,17 @@ macro(llvm_test_verify)
> set(TESTSCRIPT "${TESTSCRIPT}VERIFY: ${JOINED_ARGUMENTS}\n")
> endif()
> endmacro()
> +
> +macro(llvm_test_metric)
> + CMAKE_PARSE_ARGUMENTS(ARGS "" "RUN_TYPE;METRIC" "" ${ARGN})
> + if(NOT DEFINED TESTSCRIPT)
> + set(TESTSCRIPT "" PARENT_SCOPE)
> + endif()
> + # ARGS_UNPARSED_ARGUMENTS is a semicolon-separated list. Change it into a
> + # whitespace-separated string.
> + string(REPLACE ";" " " JOINED_ARGUMENTS "${ARGS_UNPARSED_ARGUMENTS}")
> + if(NOT DEFINED ARGS_RUN_TYPE OR "${ARGS_RUN_TYPE}" STREQUAL "${TEST_SUITE_RUN_TYPE}")
> + set(TESTSCRIPT "${TESTSCRIPT}METRIC: ${ARGS_METRIC}: ${JOINED_ARGUMENTS}\n")
> + endif()
> +endmacro()
> +
> \ No newline at end of file
>
> Modified: test-suite/trunk/litsupport/test.py
> URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/test.py?rev=261857&r1=261856&r2=261857&view=diff
> ==============================================================================
> --- test-suite/trunk/litsupport/test.py (original)
> +++ test-suite/trunk/litsupport/test.py Thu Feb 25 05:06:15 2016
> @@ -56,7 +56,7 @@ class TestSuiteTest(FileBasedTest):
> res = testscript.parse(test.getSourcePath())
> if litConfig.noExecute:
> return lit.Test.Result(Test.PASS)
> - runscript, verifyscript = res
> + runscript, verifyscript, metricscripts = res
>
> # Apply the usual lit substitutions (%s, %S, %p, %T, ...)
> tmpDir, tmpBase = getTempPaths(test)
> @@ -65,6 +65,8 @@ class TestSuiteTest(FileBasedTest):
> substitutions += [('%o', outfile)]
> runscript = applySubstitutions(runscript, substitutions)
> verifyscript = applySubstitutions(verifyscript, substitutions)
> + metricscripts = {k: applySubstitutions(v, substitutions)
> + for k,v in metricscripts.items()}
> context = TestContext(test, litConfig, runscript, verifyscript, tmpDir,
> tmpBase)
>
> @@ -80,6 +82,7 @@ class TestSuiteTest(FileBasedTest):
> output = ""
> n_runs = 1
> runtimes = []
> + metrics = {}
> for n in range(n_runs):
> res = runScript(context, runscript)
> if isinstance(res, lit.Test.Result):
> @@ -94,6 +97,15 @@ class TestSuiteTest(FileBasedTest):
> output += "\n" + err
> return lit.Test.Result(Test.FAIL, output)
>
> + # Execute metric extraction scripts.
> + for metric, script in metricscripts.items():
> + res = runScript(context, script)
> + if isinstance(res, lit.Test.Result):
> + return res
> +
> + out, err, exitCode, timeoutInfo = res
> + metrics.setdefault(metric, list()).append(float(out))
> +
> try:
> runtime = runsafely.getTime(context)
> runtimes.append(runtime)
> @@ -128,6 +140,8 @@ class TestSuiteTest(FileBasedTest):
> result = lit.Test.Result(Test.PASS, output)
> if len(runtimes) > 0:
> result.addMetric('exec_time', lit.Test.toMetricValue(runtimes[0]))
> + for metric, values in metrics.items():
> + result.addMetric(metric, lit.Test.toMetricValue(values[0]))
> compiletime.collect(context, result)
>
> return result
>
> Modified: test-suite/trunk/litsupport/testscript.py
> URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/testscript.py?rev=261857&r1=261856&r2=261857&view=diff
> ==============================================================================
> --- test-suite/trunk/litsupport/testscript.py (original)
> +++ test-suite/trunk/litsupport/testscript.py Thu Feb 25 05:06:15 2016
> @@ -22,13 +22,18 @@ def parse(filename):
> # Collect the test lines from the script.
> runscript = []
> verifyscript = []
> - keywords = ['RUN:', 'VERIFY:']
> + metricscripts = {}
> + keywords = ['RUN:', 'VERIFY:', 'METRIC:']
> for line_number, command_type, ln in \
> parseIntegratedTestScriptCommands(filename, keywords):
> if command_type == 'RUN':
> _parseShellCommand(runscript, ln)
> elif command_type == 'VERIFY':
> _parseShellCommand(verifyscript, ln)
> + elif command_type == 'METRIC':
> + metric, ln = ln.split(':', 1)
> + metricscript = metricscripts.setdefault(metric.strip(), list())
> + _parseShellCommand(metricscript, ln)
> else:
> raise ValueError("unknown script command type: %r" % (
> command_type,))
> @@ -43,4 +48,4 @@ def parse(filename):
> raise ValueError("Test has unterminated RUN/VERIFY lines " +
> "(ending with '\\')")
>
> - return runscript, verifyscript
> + return runscript, verifyscript, metricscripts
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list