[test-suite] r266779 - litsupport: Refactor parsing code, add 'run' module
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 19 10:49:09 PDT 2016
Author: matze
Date: Tue Apr 19 12:49:09 2016
New Revision: 266779
URL: http://llvm.org/viewvc/llvm-project?rev=266779&view=rev
Log:
litsupport: Refactor parsing code, add 'run' module
The run module runs the benchmark. The trick is that you can leave it
out to not run the benchmark. This is usefull in situations where
collecting codesize, compiletime and executable hash is enough.
Added:
test-suite/trunk/litsupport/run.py
test-suite/trunk/litsupport/testfile.py
Modified:
test-suite/trunk/litsupport/perf.py
test-suite/trunk/litsupport/shellcommand.py
test-suite/trunk/litsupport/test.py
test-suite/trunk/litsupport/testplan.py
Modified: test-suite/trunk/litsupport/perf.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/perf.py?rev=266779&r1=266778&r2=266779&view=diff
==============================================================================
--- test-suite/trunk/litsupport/perf.py (original)
+++ test-suite/trunk/litsupport/perf.py Tue Apr 19 12:49:09 2016
@@ -15,10 +15,9 @@ def mutateCommandLine(context, commandli
def mutatePlan(context, plan):
+ script = context.parsed_runscript
if context.config.run_under:
- script = testplan.mutateScript(context, context.original_runscript,
+ script = testplan.mutateScript(context, script,
run_under.mutateCommandLine)
- else:
- script = context.original_runscript
script = testplan.mutateScript(context, script, mutateCommandLine)
plan.profilescript += script
Added: test-suite/trunk/litsupport/run.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/run.py?rev=266779&view=auto
==============================================================================
--- test-suite/trunk/litsupport/run.py (added)
+++ test-suite/trunk/litsupport/run.py Tue Apr 19 12:49:09 2016
@@ -0,0 +1,6 @@
+def mutatePlan(context, plan):
+ """The most basic test module: Execute the RUN:, VERIFY: and METRIC:
+ scripts"""
+ plan.runscript = context.parsed_runscript
+ plan.verifyscript = context.parsed_verifyscript
+ plan.metricscripts = context.parsed_metricscripts
Modified: test-suite/trunk/litsupport/shellcommand.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/shellcommand.py?rev=266779&r1=266778&r2=266779&view=diff
==============================================================================
--- test-suite/trunk/litsupport/shellcommand.py (original)
+++ test-suite/trunk/litsupport/shellcommand.py Tue Apr 19 12:49:09 2016
@@ -122,7 +122,7 @@ def getMainExecutable(context):
return context.executable
executable = None
- for line in context.original_runscript:
+ for line in context.parsed_runscript:
cmd = parse(line)
if cmd.executable in _ignore_executables:
continue
Modified: test-suite/trunk/litsupport/test.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/test.py?rev=266779&r1=266778&r2=266779&view=diff
==============================================================================
--- test-suite/trunk/litsupport/test.py (original)
+++ test-suite/trunk/litsupport/test.py Tue Apr 19 12:49:09 2016
@@ -3,8 +3,7 @@ import lit
import lit.util
import logging
from lit.formats import FileBasedTest
-from lit.TestRunner import getDefaultSubstitutions, applySubstitutions, \
- getTempPaths
+from lit.TestRunner import getTempPaths
from lit import Test
from lit.util import to_bytes, to_string
@@ -14,8 +13,9 @@ from litsupport import hash
from litsupport import perf
from litsupport import profilegen
from litsupport import remote
+from litsupport import run
from litsupport import run_under
-from litsupport import shellcommand
+from litsupport import testfile
from litsupport import testplan
from litsupport import timeit
@@ -28,13 +28,10 @@ class TestContext:
For example this can be used by modules modifying the commandline with
extra instrumentation/measurement wrappers to pass the filenames of the
results to a final data collection step."""
- def __init__(self, test, litConfig, original_runscript,
- original_verifyscript, tmpDir, tmpBase):
+ def __init__(self, test, litConfig, tmpDir, tmpBase):
self.test = test
self.config = test.config
self.litConfig = litConfig
- self.original_runscript = original_runscript
- self.original_verifyscript = original_verifyscript
self.tmpDir = tmpDir
self.tmpBase = tmpBase
@@ -47,37 +44,28 @@ class TestSuiteTest(FileBasedTest):
config = test.config
if config.unsupported:
return lit.Test.Result(Test.UNSUPPORTED, 'Test is unsupported')
-
- # Parse benchmark script
- plan = testplan.parse(test.getSourcePath())
if litConfig.noExecute:
return lit.Test.Result(Test.PASS)
- # Apply the usual lit substitutions (%s, %S, %p, %T, ...)
+ # Parse .test file and initialize context
tmpDir, tmpBase = getTempPaths(test)
- outfile = tmpBase + ".out"
- substitutions = getDefaultSubstitutions(test, tmpDir, tmpBase)
- substitutions += [('%o', outfile)]
- plan.runscript = applySubstitutions(plan.runscript, substitutions)
- plan.verifyscript = applySubstitutions(plan.verifyscript,
- substitutions)
- plan.metricscripts = {k: applySubstitutions(v, substitutions)
- for k, v in plan.metricscripts.items()}
- context = TestContext(test, litConfig, plan.runscript,
- plan.verifyscript, tmpDir, tmpBase)
- context.executable = shellcommand.getMainExecutable(context)
- if context.executable is None:
- return lit.Test.Result(Test.UNSUPPORTED,
- 'Could not determine executable name')
- hash.compute(context)
- if hash.same_as_previous(context):
- return lit.Test.Result(SKIPPED,
- 'Executable identical to previous run')
-
- # Create the output directory if it does not already exist.
lit.util.mkdir_p(os.path.dirname(tmpBase))
+ context = TestContext(test, litConfig, tmpDir, tmpBase)
+ testfile.parse(context, test.getSourcePath())
+ plan = testplan.TestPlan()
+
+ # Skip unchanged tests
+ if config.previous_results:
+ hash.compute(context)
+ if hash.same_as_previous(context):
+ result = lit.Test.Result(
+ SKIPPED, 'Executable identical to previous run')
+ val = lit.Test.toMetricValue(context.executable_hash)
+ result.addMetric('hash', val)
+ return result
# Prepare test plan
+ run.mutatePlan(context, plan)
run_under.mutatePlan(context, plan)
timeit.mutatePlan(context, plan)
compiletime.mutatePlan(context, plan)
Added: test-suite/trunk/litsupport/testfile.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/testfile.py?rev=266779&view=auto
==============================================================================
--- test-suite/trunk/litsupport/testfile.py (added)
+++ test-suite/trunk/litsupport/testfile.py Tue Apr 19 12:49:09 2016
@@ -0,0 +1,70 @@
+"""
+Parse a .test file
+"""
+from lit.TestRunner import parseIntegratedTestScriptCommands, \
+ getDefaultSubstitutions, applySubstitutions
+from litsupport import shellcommand
+
+
+def _parseShellCommand(script, ln):
+ # Trim trailing whitespace.
+ ln = ln.rstrip()
+
+ # Collapse lines with trailing '\\'.
+ if script and script[-1][-1] == '\\':
+ script[-1] = script[-1][:-1] + ln
+ else:
+ script.append(ln)
+
+
+def parse(context, filename):
+ """Parse a .test file as used in the llvm test-suite.
+ The file comporises of a number of lines starting with RUN: and VERIFY:
+ specifying shell commands to run the benchmark and verifying the result.
+ Returns a tuple with two arrays for the run and verify commands."""
+ # Collect the test lines from the script.
+ runscript = []
+ verifyscript = []
+ metricscripts = {}
+ keywords = ['RUN:', 'VERIFY:', 'METRIC:']
+ for line_number, command_type, ln in \
+ parseIntegratedTestScriptCommands(filename, keywords):
+ if command_type == 'RUN':
+ _parseShellCommand(runscript, ln)
+ elif command_type == 'VERIFY':
+ _parseShellCommand(verifyscript, ln)
+ elif command_type == 'METRIC':
+ metric, ln = ln.split(':', 1)
+ metricscript = metricscripts.setdefault(metric.strip(), list())
+ _parseShellCommand(metricscript, ln)
+ else:
+ raise ValueError("unknown script command type: %r" % (
+ command_type,))
+
+ # Verify the script contains a run line.
+ if runscript == []:
+ raise ValueError("Test has no RUN: line!")
+
+ # Check for unterminated run lines.
+ for script in runscript, verifyscript:
+ if script and script[-1][-1] == '\\':
+ raise ValueError("Test has unterminated RUN/VERIFY lines " +
+ "(ending with '\\')")
+
+ # Apply the usual lit substitutions (%s, %S, %p, %T, ...)
+ outfile = context.tmpBase + ".out"
+ substitutions = getDefaultSubstitutions(context.test, context.tmpDir,
+ context.tmpBase)
+ substitutions += [('%o', outfile)]
+ runscript = applySubstitutions(runscript, substitutions)
+ verifyscript = applySubstitutions(verifyscript, substitutions)
+ metricscripts = {k: applySubstitutions(v, substitutions)
+ for k, v in metricscripts.items()}
+
+ # Put things into the context
+ context.parsed_runscript = runscript
+ context.parsed_verifyscript = verifyscript
+ context.parsed_metricscripts = metricscripts
+ context.executable = shellcommand.getMainExecutable(context)
+ if not context.executable:
+ logging.error("Could not determine executable name in %s" % filename)
Modified: test-suite/trunk/litsupport/testplan.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/testplan.py?rev=266779&r1=266778&r2=266779&view=diff
==============================================================================
--- test-suite/trunk/litsupport/testplan.py (original)
+++ test-suite/trunk/litsupport/testplan.py Tue Apr 19 12:49:09 2016
@@ -1,7 +1,6 @@
"""
Datastructures for test plans; Parsing of .test files; Executing test plans.
"""
-from lit.TestRunner import parseIntegratedTestScriptCommands
from litsupport import shellcommand
import lit.Test
import lit.TestRunner
@@ -11,10 +10,10 @@ import subprocess
class TestPlan(object):
- def __init__(self, runscript, verifyscript, metricscripts):
- self.runscript = runscript
- self.verifyscript = verifyscript
- self.metricscripts = metricscripts
+ def __init__(self):
+ self.runscript = []
+ self.verifyscript = []
+ self.metricscripts = {}
self.metric_collectors = []
self.profilescript = []
@@ -35,54 +34,6 @@ def mutateScript(context, script, mutato
return mutated_script
-def _parseShellCommand(script, ln):
- # Trim trailing whitespace.
- ln = ln.rstrip()
-
- # Collapse lines with trailing '\\'.
- if script and script[-1][-1] == '\\':
- script[-1] = script[-1][:-1] + ln
- else:
- script.append(ln)
-
-
-def parse(filename):
- """Parse a .test file as used in the llvm test-suite.
- The file comporises of a number of lines starting with RUN: and VERIFY:
- specifying shell commands to run the benchmark and verifying the result.
- Returns a tuple with two arrays for the run and verify commands."""
- # Collect the test lines from the script.
- runscript = []
- verifyscript = []
- metricscripts = {}
- keywords = ['RUN:', 'VERIFY:', 'METRIC:']
- for line_number, command_type, ln in \
- parseIntegratedTestScriptCommands(filename, keywords):
- if command_type == 'RUN':
- _parseShellCommand(runscript, ln)
- elif command_type == 'VERIFY':
- _parseShellCommand(verifyscript, ln)
- elif command_type == 'METRIC':
- metric, ln = ln.split(':', 1)
- metricscript = metricscripts.setdefault(metric.strip(), list())
- _parseShellCommand(metricscript, ln)
- else:
- raise ValueError("unknown script command type: %r" % (
- command_type,))
-
- # Verify the script contains a run line.
- if runscript == []:
- raise ValueError("Test has no RUN: line!")
-
- # Check for unterminated run lines.
- for script in runscript, verifyscript:
- if script and script[-1][-1] == '\\':
- raise ValueError("Test has unterminated RUN/VERIFY lines " +
- "(ending with '\\')")
-
- return TestPlan(runscript, verifyscript, metricscripts)
-
-
def executeScript(context, script, useExternalSh=True):
if len(script) == 0:
return "", "", 0, None
More information about the llvm-commits
mailing list