[cfe-commits] r77665 - in /cfe/trunk: test/CMakeLists.txt test/Makefile test/lit.cfg utils/test/MultiTestRunner.py utils/test/TestRunner.py utils/test/TestingConfig.py utils/test/Util.py
Daniel Dunbar
daniel at zuster.org
Thu Jul 30 22:54:17 PDT 2009
Author: ddunbar
Date: Fri Jul 31 00:54:17 2009
New Revision: 77665
URL: http://llvm.org/viewvc/llvm-project?rev=77665&view=rev
Log:
MultiTestRunner: Simplify, cleanup, and rename!
- MultiTestRunner will eventually be renamed to 'lit', for LLVM integrated
tester/testing. This has the pros of being pronouncable and short.
- "Project" level configuration lives in 'lit.cfg', which is also what lit uses
to find the root testing directory in some cases. This can be overridden for
use in project files which want to precisely specify where things are.
- TestRunner.py is not longer able to be invoked directly.
- Moved some code to Util.py.
- Introduced a configuration object.
- Cleaned up --help, removed a few not-very-useful options.
- Tried not to break anything that works. :)
Added:
cfe/trunk/test/lit.cfg
cfe/trunk/utils/test/TestingConfig.py
cfe/trunk/utils/test/Util.py
Modified:
cfe/trunk/test/CMakeLists.txt
cfe/trunk/test/Makefile
cfe/trunk/utils/test/MultiTestRunner.py
cfe/trunk/utils/test/TestRunner.py
Modified: cfe/trunk/test/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CMakeLists.txt?rev=77665&r1=77664&r2=77665&view=diff
==============================================================================
--- cfe/trunk/test/CMakeLists.txt (original)
+++ cfe/trunk/test/CMakeLists.txt Fri Jul 31 00:54:17 2009
@@ -33,6 +33,7 @@
add_custom_target(clang-test-${testdir}
${PYTHON_EXECUTABLE}
${LLVM_SOURCE_DIR}/tools/clang/utils/test/MultiTestRunner.py
+ "--root=${LLVM_SOURCE_DIR}/tools/clang/test"
"--path=${LLVM_TOOLS_PATH}/${CMAKE_CFG_INTDIR}"
"--path=${LLVM_SOURCE_DIR}/test/Scripts"
-s ${CLANG_TEST_EXTRA_ARGS}
@@ -48,6 +49,7 @@
add_custom_target(clang-test
${PYTHON_EXECUTABLE}
${LLVM_SOURCE_DIR}/tools/clang/utils/test/MultiTestRunner.py
+ "--root=${LLVM_SOURCE_DIR}/tools/clang/test"
"--path=${LLVM_TOOLS_PATH}/${CMAKE_CFG_INTDIR}"
"--path=${LLVM_SOURCE_DIR}/test/Scripts"
-s ${CLANG_TEST_EXTRA_ARGS}
Modified: cfe/trunk/test/Makefile
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Makefile?rev=77665&r1=77664&r2=77665&view=diff
==============================================================================
--- cfe/trunk/test/Makefile (original)
+++ cfe/trunk/test/Makefile Fri Jul 31 00:54:17 2009
@@ -21,6 +21,7 @@
all::
@ echo '--- Running clang tests for $(TARGET_TRIPLE) ---'
@ $(PROJ_SRC_DIR)/../utils/test/MultiTestRunner.py \
+ --root $(PROJ_SRC_DIR) \
--path $(ToolDir) \
--path $(LLVM_SRC_ROOT)/test/Scripts \
$(TESTARGS) $(TESTDIRS) $(VGARG)
Added: cfe/trunk/test/lit.cfg
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/lit.cfg?rev=77665&view=auto
==============================================================================
--- cfe/trunk/test/lit.cfg (added)
+++ cfe/trunk/test/lit.cfg Fri Jul 31 00:54:17 2009
@@ -0,0 +1,12 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+# suffixes: A list of file extensions to treat as test files.
+suffixes = ['.c', '.cpp']
+
+# environment: The base environment to use when running test commands.
+#
+# The 'PATH' and 'SYSTEMROOT' variables will be set automatically from the lit
+# command line variables.
+environment = {}
Modified: cfe/trunk/utils/test/MultiTestRunner.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/test/MultiTestRunner.py?rev=77665&r1=77664&r2=77665&view=diff
==============================================================================
--- cfe/trunk/utils/test/MultiTestRunner.py (original)
+++ cfe/trunk/utils/test/MultiTestRunner.py Fri Jul 31 00:54:17 2009
@@ -15,33 +15,40 @@
in a separate category).
"""
-# TOD
import os, sys, re, random, time
import threading
+from Queue import Queue
+
import ProgressBar
import TestRunner
+import Util
+
+from TestingConfig import TestingConfig
from TestRunner import TestStatus
-from Queue import Queue
-kTestFileExtensions = set(['.mi','.i','.c','.cpp','.m','.mm','.ll'])
+kConfigName = 'lit.cfg'
-def getTests(inputs):
+def getTests(cfg, inputs):
for path in inputs:
if not os.path.exists(path):
- print >>sys.stderr,"WARNING: Invalid test \"%s\""%(path,)
+ Util.warning('Invalid test %r' % path)
continue
if not os.path.isdir(path):
yield path
+ foundOne = False
for dirpath,dirnames,filenames in os.walk(path):
# FIXME: This doesn't belong here
if 'Output' in dirnames:
dirnames.remove('Output')
for f in filenames:
base,ext = os.path.splitext(f)
- if ext in kTestFileExtensions:
+ if ext in cfg.suffixes:
yield os.path.join(dirpath,f)
+ foundOne = True
+ if not foundOne:
+ Util.warning('No tests in input directory %r' % path)
class TestingProgressDisplay:
def __init__(self, opts, numTests, progressBar=None):
@@ -109,7 +116,8 @@
return self.code in (TestStatus.Fail,TestStatus.XPass)
class TestProvider:
- def __init__(self, opts, tests, display):
+ def __init__(self, config, opts, tests, display):
+ self.config = config
self.opts = opts
self.tests = tests
self.index = 0
@@ -156,14 +164,12 @@
elapsed = None
try:
opts = self.provider.opts
- if opts.debugDoNotTest:
- code = None
- else:
- startTime = time.time()
- code, output = TestRunner.runOneTest(path, base,
- opts.clang, opts.clangcc,
- opts.useValgrind)
- elapsed = time.time() - startTime
+ startTime = time.time()
+ code, output = TestRunner.runOneTest(self.provider.config,
+ path, base,
+ opts.clang, opts.clangcc,
+ opts.useValgrind)
+ elapsed = time.time() - startTime
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
@@ -172,102 +178,127 @@
self.provider.setResult(index, TestResult(path, code, output, elapsed))
-def detectCPUs():
- """
- Detects the number of CPUs on a system. Cribbed from pp.
- """
- # Linux, Unix and MacOS:
- if hasattr(os, "sysconf"):
- if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
- # Linux & Unix:
- ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
- if isinstance(ncpus, int) and ncpus > 0:
- return ncpus
- else: # OSX:
- return int(os.popen2("sysctl -n hw.ncpu")[1].read())
- # Windows:
- if os.environ.has_key("NUMBER_OF_PROCESSORS"):
- ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]);
- if ncpus > 0:
- return ncpus
- return 1 # Default
+def findConfigPath(root):
+ prev = None
+ while root != prev:
+ cfg = os.path.join(root, kConfigName)
+ if os.path.exists(cfg):
+ return cfg
+
+ prev,root = root,os.path.dirname(root)
+
+ raise ValueError,"Unable to find config file %r" % kConfigName
def main():
global options
- from optparse import OptionParser
- parser = OptionParser("usage: %prog [options] {inputs}")
- parser.add_option("-j", "--threads", dest="numThreads",
- help="Number of testing threads",
- type=int, action="store",
- default=detectCPUs())
- parser.add_option("", "--clang", dest="clang",
- help="Program to use as \"clang\"",
- action="store", default=None)
- parser.add_option("", "--clang-cc", dest="clangcc",
- help="Program to use as \"clang-cc\"",
- action="store", default=None)
- parser.add_option("", "--vg", dest="useValgrind",
- help="Run tests under valgrind",
- action="store_true", default=False)
- parser.add_option("-v", "--verbose", dest="showOutput",
- help="Show all test output",
- action="store_true", default=False)
- parser.add_option("-q", "--quiet", dest="quiet",
- help="Suppress no error output",
- action="store_true", default=False)
- parser.add_option("-s", "--succinct", dest="succinct",
- help="Reduce amount of output",
- action="store_true", default=False)
- parser.add_option("", "--max-tests", dest="maxTests",
- help="Maximum number of tests to run",
- action="store", type=int, default=None)
- parser.add_option("", "--max-time", dest="maxTime",
- help="Maximum time to spend testing (in seconds)",
- action="store", type=float, default=None)
- parser.add_option("", "--shuffle", dest="shuffle",
- help="Run tests in random order",
- action="store_true", default=False)
- parser.add_option("", "--seed", dest="seed",
- help="Seed for random number generator (default: random)",
+ from optparse import OptionParser, OptionGroup
+ parser = OptionParser("usage: %prog [options] {file-or-path}")
+
+ parser.add_option("", "--root", dest="root",
+ help="Path to root test directory",
action="store", default=None)
- parser.add_option("", "--no-progress-bar", dest="useProgressBar",
- help="Do not use curses based progress bar",
- action="store_false", default=True)
- parser.add_option("", "--debug-do-not-test", dest="debugDoNotTest",
- help="DEBUG: Skip running actual test script",
- action="store_true", default=False)
- parser.add_option("", "--time-tests", dest="timeTests",
- help="Track elapsed wall time for each test",
- action="store_true", default=False)
- parser.add_option("", "--path", dest="path",
- help="Additional paths to add to testing environment",
- action="append", type=str, default=[])
+ parser.add_option("", "--config", dest="config",
+ help="Testing configuration file [default='%default']",
+ action="store", default=kConfigName)
+
+ group = OptionGroup(parser, "Output Format")
+ # FIXME: I find these names very confusing, although I like the
+ # functionality.
+ group.add_option("-q", "--quiet", dest="quiet",
+ help="Suppress no error output",
+ action="store_true", default=False)
+ group.add_option("-s", "--succinct", dest="succinct",
+ help="Reduce amount of output",
+ action="store_true", default=False)
+ group.add_option("-v", "--verbose", dest="showOutput",
+ help="Show all test output",
+ action="store_true", default=False)
+ group.add_option("", "--no-progress-bar", dest="useProgressBar",
+ help="Do not use curses based progress bar",
+ action="store_false", default=True)
+ parser.add_option_group(group)
+
+ group = OptionGroup(parser, "Test Execution")
+ group.add_option("-j", "--threads", dest="numThreads",
+ help="Number of testing threads",
+ type=int, action="store",
+ default=None)
+ group.add_option("", "--clang", dest="clang",
+ help="Program to use as \"clang\"",
+ action="store", default=None)
+ group.add_option("", "--clang-cc", dest="clangcc",
+ help="Program to use as \"clang-cc\"",
+ action="store", default=None)
+ group.add_option("", "--path", dest="path",
+ help="Additional paths to add to testing environment",
+ action="append", type=str, default=[])
+ group.add_option("", "--vg", dest="useValgrind",
+ help="Run tests under valgrind",
+ action="store_true", default=False)
+ group.add_option("", "--time-tests", dest="timeTests",
+ help="Track elapsed wall time for each test",
+ action="store_true", default=False)
+ parser.add_option_group(group)
+
+ group = OptionGroup(parser, "Test Selection")
+ group.add_option("", "--max-tests", dest="maxTests",
+ help="Maximum number of tests to run",
+ action="store", type=int, default=None)
+ group.add_option("", "--max-time", dest="maxTime",
+ help="Maximum time to spend testing (in seconds)",
+ action="store", type=float, default=None)
+ group.add_option("", "--shuffle", dest="shuffle",
+ help="Run tests in random order",
+ action="store_true", default=False)
+ parser.add_option_group(group)
(opts, args) = parser.parse_args()
-
+
if not args:
parser.error('No inputs specified')
- # FIXME: Move into configuration object.
- TestRunner.kChildEnv["PATH"] = os.pathsep.join(opts.path +
- [TestRunner.kChildEnv['PATH']])
+ if opts.numThreads is None:
+ opts.numThreads = Util.detectCPUs()
+
+ inputs = args
+
+ # Resolve root if not given, either infer it from the config file if given,
+ # otherwise from the inputs.
+ if not opts.root:
+ if opts.config:
+ opts.root = os.path.dirname(opts.config)
+ else:
+ opts.root = os.path.commonprefix(inputs)
+
+ # Find the config file, if not specified.
+ if not opts.config:
+ try:
+ opts.config = findConfigPath(opts.root)
+ except ValueError,e:
+ parser.error(e.args[0])
+
+ cfg = TestingConfig.frompath(opts.config)
+
+ # Update the configuration based on the command line arguments.
+ for name in ('PATH','SYSTEMROOT'):
+ if name in cfg.environment:
+ parser.error("'%s' should not be set in configuration!" % name)
+
+ cfg.root = opts.root
+ cfg.environment['PATH'] = os.pathsep.join(opts.path +
+ [os.environ.get('PATH','')])
+ cfg.environment['SYSTEMROOT'] = os.environ.get('SYSTEMROOT','')
if opts.clang is None:
- opts.clang = TestRunner.inferClang()
+ opts.clang = TestRunner.inferClang(cfg)
if opts.clangcc is None:
- opts.clangcc = TestRunner.inferClangCC(opts.clang)
+ opts.clangcc = TestRunner.inferClangCC(cfg, opts.clang)
# FIXME: It could be worth loading these in parallel with testing.
- allTests = list(getTests(args))
+ allTests = list(getTests(cfg, args))
allTests.sort()
tests = allTests
- if opts.seed is not None:
- try:
- seed = int(opts.seed)
- except:
- parser.error('--seed argument should be an integer')
- random.seed(seed)
if opts.shuffle:
random.shuffle(tests)
if opts.maxTests is not None:
@@ -292,7 +323,7 @@
print header
display = TestingProgressDisplay(opts, len(tests), progressBar)
- provider = TestProvider(opts, tests, display)
+ provider = TestProvider(cfg, opts, tests, display)
testers = [Tester(provider) for i in range(opts.numThreads)]
startTime = time.time()
@@ -309,7 +340,7 @@
if not opts.quiet:
print 'Testing Time: %.2fs'%(time.time() - startTime)
- # List test results organized organized by kind.
+ # List test results organized by kind.
byCode = {}
for t in provider.results:
if t:
Modified: cfe/trunk/utils/test/TestRunner.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/test/TestRunner.py?rev=77665&r1=77664&r2=77665&view=diff
==============================================================================
--- cfe/trunk/utils/test/TestRunner.py (original)
+++ cfe/trunk/utils/test/TestRunner.py Fri Jul 31 00:54:17 2009
@@ -23,10 +23,7 @@
import subprocess
import sys
-# Increase determinism by explicitly choosing the environment.
-kChildEnv = {}
-for var in ('PATH', 'SYSTEMROOT'):
- kChildEnv[var] = os.environ.get(var, '')
+import Util
kSystemName = platform.system()
@@ -42,22 +39,7 @@
def getName(code):
return TestStatus.kNames[code]
-def mkdir_p(path):
- if not path:
- pass
- elif os.path.exists(path):
- pass
- else:
- parent = os.path.dirname(path)
- if parent != path:
- mkdir_p(parent)
- try:
- os.mkdir(path)
- except OSError,e:
- if e.errno != errno.EEXIST:
- raise
-
-def executeScript(script, commands, cwd, useValgrind):
+def executeScript(cfg, script, commands, cwd, useValgrind):
# Write script file
f = open(script,'w')
if kSystemName == 'Windows':
@@ -82,7 +64,7 @@
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- env=kChildEnv)
+ env=cfg.environment)
out,err = p.communicate()
exitCode = p.wait()
@@ -93,14 +75,14 @@
return out, err, exitCode
import StringIO
-def runOneTest(testPath, tmpBase, clang, clangcc, useValgrind):
+def runOneTest(cfg, testPath, tmpBase, clang, clangcc, useValgrind):
# Make paths absolute.
tmpBase = os.path.abspath(tmpBase)
testPath = os.path.abspath(testPath)
# Create the output directory if it does not already exist.
- mkdir_p(os.path.dirname(tmpBase))
+ Util.mkdir_p(os.path.dirname(tmpBase))
script = tmpBase + '.script'
if kSystemName == 'Windows':
script += '.bat'
@@ -154,7 +136,7 @@
# Strip off '&&'
scriptLines[i] = ln[:-2]
- out, err, exitCode = executeScript(script, scriptLines,
+ out, err, exitCode = executeScript(cfg, script, scriptLines,
os.path.dirname(testPath),
useValgrind)
if xfailLines:
@@ -188,31 +170,7 @@
out,_ = p.communicate()
return out
-def which(command):
- # FIXME: Take configuration object.
-
- # Check for absolute match first.
- if os.path.exists(command):
- return command
-
- # Would be nice if Python had a lib function for this.
- paths = kChildEnv['PATH']
- if not paths:
- paths = os.defpath
-
- # Get suffixes to search.
- pathext = os.environ.get('PATHEXT', '').split(os.pathsep)
-
- # Search the paths...
- for path in paths.split(os.pathsep):
- for ext in pathext:
- p = os.path.join(path, command + ext)
- if os.path.exists(p):
- return p
-
- return None
-
-def inferClang():
+def inferClang(cfg):
# Determine which clang to use.
clang = os.getenv('CLANG')
@@ -222,7 +180,7 @@
return clang
# Otherwise look in the path.
- clang = which('clang')
+ clang = Util.which('clang', cfg.environment['PATH'])
if not clang:
print >>sys.stderr, "error: couldn't find 'clang' program, try setting CLANG in your environment"
@@ -230,7 +188,7 @@
return clang
-def inferClangCC(clang):
+def inferClangCC(cfg, clang):
clangcc = os.getenv('CLANGCC')
# If the user set clang in the environment, definitely use that and don't
@@ -244,7 +202,7 @@
clangccName = clang[:-4] + '-cc.exe'
else:
clangccName = clang + '-cc'
- clangcc = which(clangccName)
+ clangcc = Util.which(clangccName, cfg.environment['PATH'])
if not clangcc:
# Otherwise ask clang.
res = capture([clang, '-print-prog-name=clang-cc'])
@@ -267,43 +225,3 @@
return os.path.join(dir,
os.path.basename(os.path.dirname(testpath)),
os.path.basename(testpath))
-
-def main():
- global options
- from optparse import OptionParser
- parser = OptionParser("usage: %prog [options] {tests}")
- parser.add_option("", "--clang", dest="clang",
- help="Program to use as \"clang\"",
- action="store", default=None)
- parser.add_option("", "--clang-cc", dest="clangcc",
- help="Program to use as \"clang-cc\"",
- action="store", default=None)
- parser.add_option("", "--vg", dest="useValgrind",
- help="Run tests under valgrind",
- action="store_true", default=False)
- (opts, args) = parser.parse_args()
-
- if not args:
- parser.error('No tests specified')
-
- if opts.clang is None:
- opts.clang = inferClang()
- if opts.clangcc is None:
- opts.clangcc = inferClangCC(opts.clang)
-
- for path in args:
- base = getTestOutputBase('Output', path) + '.out'
-
- status,output = runOneTest(path, base, opts.clang, opts.clangcc,
- opts.useValgrind)
- print '%s: %s' % (TestStatus.getName(status).upper(), path)
- if status == TestStatus.Fail or status == TestStatus.XPass:
- print "%s TEST '%s' FAILED %s" % ('*'*20, path, '*'*20)
- sys.stdout.write(output)
- print "*" * 20
- sys.exit(1)
-
- sys.exit(0)
-
-if __name__=='__main__':
- main()
Added: cfe/trunk/utils/test/TestingConfig.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/test/TestingConfig.py?rev=77665&view=auto
==============================================================================
--- cfe/trunk/utils/test/TestingConfig.py (added)
+++ cfe/trunk/utils/test/TestingConfig.py Fri Jul 31 00:54:17 2009
@@ -0,0 +1,21 @@
+class TestingConfig:
+ """"
+ TestingConfig - Information on a how to run a group of tests.
+ """
+
+ @staticmethod
+ def frompath(path):
+ data = {}
+ f = open(path)
+ exec f in {},data
+
+ return TestingConfig(suffixes = data.get('suffixes', []),
+ environment = data.get('environment', {}))
+
+ def __init__(self, suffixes, environment):
+ self.root = None
+ self.suffixes = set(suffixes)
+ self.environment = dict(environment)
+
+
+
Added: cfe/trunk/utils/test/Util.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/test/Util.py?rev=77665&view=auto
==============================================================================
--- cfe/trunk/utils/test/Util.py (added)
+++ cfe/trunk/utils/test/Util.py Fri Jul 31 00:54:17 2009
@@ -0,0 +1,70 @@
+import os, sys
+
+def warning(msg):
+ print >>sys.stderr, '%s: warning: %s' % (sys.argv[0], msg)
+
+def detectCPUs():
+ """
+ Detects the number of CPUs on a system. Cribbed from pp.
+ """
+ # Linux, Unix and MacOS:
+ if hasattr(os, "sysconf"):
+ if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
+ # Linux & Unix:
+ ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
+ if isinstance(ncpus, int) and ncpus > 0:
+ return ncpus
+ else: # OSX:
+ return int(os.popen2("sysctl -n hw.ncpu")[1].read())
+ # Windows:
+ if os.environ.has_key("NUMBER_OF_PROCESSORS"):
+ ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]);
+ if ncpus > 0:
+ return ncpus
+ return 1 # Default
+
+def mkdir_p(path):
+ """mkdir_p(path) - Make the "path" directory, if it does not exist; this
+ will also make directories for any missing parent directories."""
+
+ if not path or os.path.exists(path):
+ return
+
+ parent = os.path.dirname(path)
+ if parent != path:
+ mkdir_p(parent)
+
+ try:
+ os.mkdir(path)
+ except OSError,e:
+ # Ignore EEXIST, which may occur during a race condition.
+ if e.errno != errno.EEXIST:
+ raise
+
+def which(command, paths = None):
+ """which(command, [paths]) - Look up the given command in the paths string (or
+ the PATH environment variable, if unspecified)."""
+
+ if paths is None:
+ paths = os.environ.get('PATH','')
+
+ # Check for absolute match first.
+ if os.path.exists(command):
+ return command
+
+ # Would be nice if Python had a lib function for this.
+ if not paths:
+ paths = os.defpath
+
+ # Get suffixes to search.
+ pathext = os.environ.get('PATHEXT', '').split(os.pathsep)
+
+ # Search the paths...
+ for path in paths.split(os.pathsep):
+ for ext in pathext:
+ p = os.path.join(path, command + ext)
+ if os.path.exists(p):
+ return p
+
+ return None
+
More information about the cfe-commits
mailing list