[LNT] r307803 - Use lnt.tests list to construct click runtest group
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 12 09:14:56 PDT 2017
Author: matze
Date: Wed Jul 12 09:14:56 2017
New Revision: 307803
URL: http://llvm.org/viewvc/llvm-project?rev=307803&view=rev
Log:
Use lnt.tests list to construct click runtest group
- We only have 1 hardcoded list of tests left in lnt.tests.__init__.py
- Avoid some duplication of the test descriptions:
We had module docstring, short_help and cli method docstring before;
reduce it to just the module docstring.
- As a bonus test suites are imported lazily now.
Modified:
lnt/trunk/lnt/lnttool/main.py
lnt/trunk/lnt/tests/compile.py
lnt/trunk/lnt/tests/nt.py
lnt/trunk/lnt/tests/test_suite.py
Modified: lnt/trunk/lnt/lnttool/main.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/lnttool/main.py?rev=307803&r1=307802&r2=307803&view=diff
==============================================================================
--- lnt/trunk/lnt/lnttool/main.py (original)
+++ lnt/trunk/lnt/lnttool/main.py Wed Jul 12 09:14:56 2017
@@ -10,9 +10,6 @@ import click
import lnt
from lnt.util import logger
import lnt.testing.profile.profile as profile
-from lnt.tests.nt import NTTest
-from lnt.tests.compile import CompileTest
-from lnt.tests.test_suite import TestSuiteTest
from .common import init_logger
from .create import action_create
@@ -117,18 +114,23 @@ def _print_result_url(results, verbose):
print "Results available at: no URL available"
- at click.group("runtest", context_settings=dict(
+class RunTestCLI(click.MultiCommand):
+ def list_commands(self, ctx):
+ import lnt.tests
+ return lnt.tests.get_names()
+
+ def get_command(self, ctx, name):
+ import lnt.tests
+ return lnt.tests.get_module(name).cli_action
+
+
+ at click.group("runtest", cls=RunTestCLI, context_settings=dict(
ignore_unknown_options=True, allow_extra_args=True,))
def action_runtest():
"""run a builtin test application"""
init_logger(logging.INFO)
-action_runtest.add_command(NTTest.cli_wrapper)
-action_runtest.add_command(CompileTest.cli_wrapper)
-action_runtest.add_command(TestSuiteTest.cli_wrapper)
-
-
@click.command("showtests")
def action_showtests():
"""show the available built-in tests"""
Modified: lnt/trunk/lnt/tests/compile.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/tests/compile.py?rev=307803&r1=307802&r2=307803&view=diff
==============================================================================
--- lnt/trunk/lnt/tests/compile.py (original)
+++ lnt/trunk/lnt/tests/compile.py Wed Jul 12 09:14:56 2017
@@ -703,118 +703,6 @@ We run each of the compile time tests in
class CompileTest(builtintest.BuiltinTest):
- # FIXME: an equivalent to argparse's add_argument_group is not implemented
- # on click. Need to review it when such functionality is available.
- # https://github.com/pallets/click/issues/373
- @staticmethod
- @click.command("compile", help=usage_info,
- short_help="Single file compile-time performance testing")
- @click.argument("label", default=platform.uname()[1], required=False,
- type=click.UNPROCESSED)
- @click.option("-s", "--sandbox", "sandbox_path", required=True,
- help="Parent directory to build and run tests in",
- type=click.UNPROCESSED, default=None, metavar="PATH")
- # Test Options
- @click.option("--no-timestamp", "timestamp_build",
- help="Don't timestamp build directory (for testing)",
- flag_value=False, default=True)
- @click.option("--cc", "cc", type=click.UNPROCESSED, required=True,
- help="Path to the compiler under test")
- @click.option("--cxx", "cxx",
- help="Path to the C++ compiler to test",
- type=click.UNPROCESSED, default=None)
- @click.option("--ld", "ld",
- help="Path to the c linker to use. (Xcode Distinction)",
- type=click.UNPROCESSED, default=None)
- @click.option("--ldxx", "ldxx",
- help="Path to the cxx linker to use. (Xcode Distinction)",
- type=click.UNPROCESSED, default=None)
- @click.option("--runn", "runn",
- help="Path to runN tool.",
- type=click.UNPROCESSED, default="runN")
- @click.option("--test-externals", "test_suite_externals", required=True,
- help="Path to the LLVM test-suite externals",
- type=click.UNPROCESSED, default=None, metavar="PATH")
- @click.option("--machine-param", "machine_parameters",
- metavar="NAME=VAL",
- help="Add 'NAME' = 'VAL' to the machine parameters",
- type=click.UNPROCESSED, multiple=True, default=[])
- @click.option("--run-param", "run_parameters",
- metavar="NAME=VAL",
- help="Add 'NAME' = 'VAL' to the run parameters",
- type=click.UNPROCESSED, multiple=True, default=[])
- @click.option("--run-order", "run_order", metavar="STR",
- help="String to use to identify and order this run",
- type=click.UNPROCESSED, default=None)
- @click.option("--test-subdir", "test_subdir",
- help="Subdirectory of test external dir to look for "
- "tests in.",
- type=click.UNPROCESSED, default="lnt-compile-suite-src")
- # Test Selection
- @click.option("--no-memory-profiling", "memory_profiling",
- help="Disable memory profiling",
- flag_value=False, default=True)
- @click.option("--multisample", "run_count", metavar="N",
- help="Accumulate test data from multiple runs",
- type=int, default=3)
- @click.option("--min-sample-time", "min_sample_time",
- help="Ensure all tests run for at least N seconds",
- metavar="N", type=float, default=.5)
- @click.option("--save-temps", "save_temps",
- help="Save temporary build output files", is_flag=True)
- @click.option("--show-tests", "show_tests",
- help="Only list the availables tests that will be run",
- is_flag=True)
- @click.option("--test", "tests", metavar="NAME",
- help="Individual test to run",
- multiple=True, default=[], type=click.UNPROCESSED)
- @click.option("--test-filter", "test_filters",
- help="Run tests matching the given pattern",
- metavar="REGEXP", multiple=True, default=[],
- type=click.UNPROCESSED)
- @click.option("--flags-to-test", "flags_to_test",
- help="Add a set of flags to test (space separated)",
- metavar="FLAGLIST", multiple=True, default=[],
- type=click.UNPROCESSED)
- @click.option("--jobs-to-test", "jobs_to_test",
- help="Add a job count to test (full builds)",
- metavar="NUM", multiple=True, default=[], type=int)
- @click.option("--config-to-test", "configs_to_test",
- help="Add build configuration to test (full builds)",
- metavar="NAME", multiple=True, default=[],
- type=click.Choice(['Debug', 'Release']))
- # Output Options
- @click.option("--no-machdep-info", "use_machdep_info",
- help=("Don't put machine (instance) dependent "
- "variables in machine info"),
- flag_value=False, default=True)
- @click.option("--machine-name", "machine_name", type=click.UNPROCESSED,
- help="Machine name to use in submission",
- default=platform.uname()[1])
- @click.option("--submit", "submit_url", metavar="URLORPATH",
- help=("autosubmit the test result to the given server "
- "(or local instance)"),
- type=click.UNPROCESSED, default=None)
- @click.option("--commit", "commit",
- help="whether the autosubmit result should be committed",
- type=int, default=True)
- @click.option("--output", "output", metavar="PATH",
- help="write raw report data to PATH (or stdout if '-')")
- @click.option("-v", "--verbose", "verbose",
- help="show verbose test results", is_flag=True)
- def cli_wrapper(*args, **kwargs):
- """Single file compile-time performance testing"""
- global opts
-
- compile_test = CompileTest()
- opts = compile_test.opts
-
- for key, value in kwargs.items():
- setattr(compile_test.opts, key, value)
-
- results = compile_test.run_test(compile_test.opts)
- compile_test.show_results_url(results)
-
def run_test(self, opts):
# Resolve the cc_under_test path.
@@ -1072,3 +960,112 @@ class CompileTest(builtintest.BuiltinTes
server_report = self.submit(lnt_report_path, opts, ts_name='compile')
return server_report
+
+# FIXME: an equivalent to argparse's add_argument_group is not implemented
+# on click. Need to review it when such functionality is available.
+# https://github.com/pallets/click/issues/373
+ at click.command("compile", help=usage_info, short_help=__doc__)
+ at click.argument("label", default=platform.uname()[1], required=False,
+ type=click.UNPROCESSED)
+ at click.option("-s", "--sandbox", "sandbox_path", required=True,
+ help="Parent directory to build and run tests in",
+ type=click.UNPROCESSED, default=None, metavar="PATH")
+# Test Options
+ at click.option("--no-timestamp", "timestamp_build",
+ help="Don't timestamp build directory (for testing)",
+ flag_value=False, default=True)
+ at click.option("--cc", "cc", type=click.UNPROCESSED, required=True,
+ help="Path to the compiler under test")
+ at click.option("--cxx", "cxx",
+ help="Path to the C++ compiler to test",
+ type=click.UNPROCESSED, default=None)
+ at click.option("--ld", "ld",
+ help="Path to the c linker to use. (Xcode Distinction)",
+ type=click.UNPROCESSED, default=None)
+ at click.option("--ldxx", "ldxx",
+ help="Path to the cxx linker to use. (Xcode Distinction)",
+ type=click.UNPROCESSED, default=None)
+ at click.option("--runn", "runn",
+ help="Path to runN tool.",
+ type=click.UNPROCESSED, default="runN")
+ at click.option("--test-externals", "test_suite_externals", required=True,
+ help="Path to the LLVM test-suite externals",
+ type=click.UNPROCESSED, default=None, metavar="PATH")
+ at click.option("--machine-param", "machine_parameters",
+ metavar="NAME=VAL",
+ help="Add 'NAME' = 'VAL' to the machine parameters",
+ type=click.UNPROCESSED, multiple=True, default=[])
+ at click.option("--run-param", "run_parameters",
+ metavar="NAME=VAL",
+ help="Add 'NAME' = 'VAL' to the run parameters",
+ type=click.UNPROCESSED, multiple=True, default=[])
+ at click.option("--run-order", "run_order", metavar="STR",
+ help="String to use to identify and order this run",
+ type=click.UNPROCESSED, default=None)
+ at click.option("--test-subdir", "test_subdir",
+ help="Subdirectory of test external dir to look for "
+ "tests in.",
+ type=click.UNPROCESSED, default="lnt-compile-suite-src")
+# Test Selection
+ at click.option("--no-memory-profiling", "memory_profiling",
+ help="Disable memory profiling",
+ flag_value=False, default=True)
+ at click.option("--multisample", "run_count", metavar="N",
+ help="Accumulate test data from multiple runs",
+ type=int, default=3)
+ at click.option("--min-sample-time", "min_sample_time",
+ help="Ensure all tests run for at least N seconds",
+ metavar="N", type=float, default=.5)
+ at click.option("--save-temps", "save_temps",
+ help="Save temporary build output files", is_flag=True)
+ at click.option("--show-tests", "show_tests",
+ help="Only list the availables tests that will be run",
+ is_flag=True)
+ at click.option("--test", "tests", metavar="NAME",
+ help="Individual test to run",
+ multiple=True, default=[], type=click.UNPROCESSED)
+ at click.option("--test-filter", "test_filters",
+ help="Run tests matching the given pattern",
+ metavar="REGEXP", multiple=True, default=[],
+ type=click.UNPROCESSED)
+ at click.option("--flags-to-test", "flags_to_test",
+ help="Add a set of flags to test (space separated)",
+ metavar="FLAGLIST", multiple=True, default=[],
+ type=click.UNPROCESSED)
+ at click.option("--jobs-to-test", "jobs_to_test",
+ help="Add a job count to test (full builds)",
+ metavar="NUM", multiple=True, default=[], type=int)
+ at click.option("--config-to-test", "configs_to_test",
+ help="Add build configuration to test (full builds)",
+ metavar="NAME", multiple=True, default=[],
+ type=click.Choice(['Debug', 'Release']))
+# Output Options
+ at click.option("--no-machdep-info", "use_machdep_info",
+ help=("Don't put machine (instance) dependent "
+ "variables in machine info"),
+ flag_value=False, default=True)
+ at click.option("--machine-name", "machine_name", type=click.UNPROCESSED,
+ help="Machine name to use in submission",
+ default=platform.uname()[1])
+ at click.option("--submit", "submit_url", metavar="URLORPATH",
+ help=("autosubmit the test result to the given server "
+ "(or local instance)"),
+ type=click.UNPROCESSED, default=None)
+ at click.option("--commit", "commit",
+ help="whether the autosubmit result should be committed",
+ type=int, default=True)
+ at click.option("--output", "output", metavar="PATH",
+ help="write raw report data to PATH (or stdout if '-')")
+ at click.option("-v", "--verbose", "verbose",
+ help="show verbose test results", is_flag=True)
+def cli_action(*args, **kwargs):
+ global opts
+
+ compile_test = CompileTest()
+ opts = compile_test.opts
+
+ for key, value in kwargs.items():
+ setattr(compile_test.opts, key, value)
+
+ results = compile_test.run_test(compile_test.opts)
+ compile_test.show_results_url(results)
Modified: lnt/trunk/lnt/tests/nt.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/tests/nt.py?rev=307803&r1=307802&r2=307803&view=diff
==============================================================================
--- lnt/trunk/lnt/tests/nt.py (original)
+++ lnt/trunk/lnt/tests/nt.py Wed Jul 12 09:14:56 2017
@@ -1497,232 +1497,6 @@ nightly test, but it should not be used
class NTTest(builtintest.BuiltinTest):
- # FIXME: an equivalent to argparse's add_argument_group is not implemented
- # on click. Need to review it when such functionality is available.
- # https://github.com/pallets/click/issues/373
- @staticmethod
- @click.command("nt", help=usage_info,
- short_help="LLVM test-suite compile and execution tests")
- @click.argument("label", default=platform.uname()[1], required=False,
- type=click.UNPROCESSED)
- # Sandbox
- @click.option("-s", "--sandbox", "sandbox_path", required=True,
- help="parent directory to build and run tests in",
- type=click.UNPROCESSED)
- @click.option("--no-timestamp", "timestamp_build", flag_value=False,
- default=True, show_default=True,
- help="don't timestamp build directory (for testing)")
- @click.option("--no-configure", "run_configure", flag_value=False,
- default=True, show_default=True,
- help="don't run configure if Makefile.config is "
- "present (only useful with --no-timestamp)")
- # Inputs
- @click.option("--without-llvm", is_flag=True, show_default=True,
- help="don't use any LLVM source or build products")
- @click.option("--llvm-src", "llvm_src_root",
- help="path to the LLVM source tree",
- type=click.UNPROCESSED)
- @click.option("--llvm-obj", "llvm_obj_root", metavar="PATH",
- help="path to the LLVM source tree",
- type=click.UNPROCESSED)
- @click.option("--test-suite", "test_suite_root", metavar="PATH",
- help="path to the LLVM test-suite sources",
- type=click.UNPROCESSED)
- @click.option("--test-externals", "test_suite_externals",
- show_default=True,
- help="path to the LLVM test-suite externals",
- default='/dev/null', metavar="PATH",
- type=click.UNPROCESSED)
- # Test Compiler
- @click.option("--cc", "cc_under_test", metavar="CC",
- help="path to the C compiler to test",
- type=click.UNPROCESSED)
- @click.option("--cxx", "cxx_under_test", metavar="CXX",
- help="path to the C++ compiler to test",
- type=click.UNPROCESSED)
- @click.option("--cc-reference",
- help="path to the reference C compiler",
- type=click.UNPROCESSED)
- @click.option("--cxx-reference",
- help="path to the reference C++ compiler",
- type=click.UNPROCESSED)
- # Test Options
- @click.option("--arch", help="Set -arch in TARGET_FLAGS")
- @click.option("--llvm-arch",
- help="Set the ARCH value used in the makefiles to",
- type=click.UNPROCESSED)
- @click.option("--make-param", "make_parameters", multiple=True,
- help="Add 'NAME' = 'VAL' to the makefile parameters",
- type=click.UNPROCESSED)
- @click.option("--isysroot", "isysroot", metavar="PATH",
- help="Set -isysroot in TARGET_FLAGS",
- type=click.UNPROCESSED)
- @click.option("--liblto-path", metavar="PATH",
- help="Specify the path to the libLTO library",
- type=click.UNPROCESSED)
- @click.option("--mcpu", metavar="CPU",
- help="Set -mcpu in TARGET_LLCFLAGS",
- type=click.UNPROCESSED)
- @click.option("--relocation-model", metavar="MODEL",
- help="Set -relocation-model in TARGET_LLCFLAGS",
- type=click.UNPROCESSED)
- @click.option("--disable-fp-elim", is_flag=True,
- help="Set -disable-fp-elim in TARGET_LLCFLAGS")
- @click.option("--optimize-option", show_default=True,
- help="Set optimization level for {LLC_,LLI_,}OPTFLAGS",
- type=click.Choice(['-O0', '-O1', '-O2',
- '-O3', '-Os', '-Oz']),
- default='-O3')
- @click.option("--cflag", "cflags", multiple=True,
- help="Additional flags to set in TARGET_FLAGS",
- type=click.UNPROCESSED, default=[], metavar="FLAG")
- @click.option("--cflags", "cflag_string", multiple=True,
- help="Additional flags to set in TARGET_FLAGS, space "
- "separated string. These flags are appended after "
- "*all* the individual --cflag arguments.",
- type=click.UNPROCESSED, default='', metavar="FLAG")
- @click.option("--mllvm", multiple=True,
- help="Add -mllvm FLAG to TARGET_FLAGS",
- type=click.UNPROCESSED, default=[], metavar="FLAG")
- @click.option("--spec-with-pgo", is_flag=True, help="Use PGO with SPEC")
- @click.option("--build-mode", metavar="NAME", show_default=True,
- default='Release+Asserts',
- help="Select the LLVM build mode to use",
- type=click.UNPROCESSED)
- @click.option("--simple", "test_simple", is_flag=True,
- help="Use TEST=simple instead of TEST=nightly")
- @click.option("--test-style", type=click.Choice(['nightly', 'simple']),
- default='simple', help="Set the test style to run")
- @click.option("--test-time-stat", type=click.Choice(['user', 'real']),
- default='user', show_default=True,
- help="Set the test timing statistic to gather")
- @click.option("--disable-cxx", "test_cxx", flag_value=False, default=True,
- show_default=True, help="Disable C++ tests")
- @click.option("--disable-externals", "test_externals", flag_value=False,
- default=True, show_default=True,
- help="Disable test suite externals (if configured)")
- @click.option("--enable-integrated-as", "test_integrated_as", is_flag=True,
- help="Enable TEST_INTEGRATED_AS tests")
- @click.option("--enable-jit", "test_jit", is_flag=True,
- help="Enable JIT tests")
- @click.option("--disable-llc", "test_llc",
- help="Disable LLC tests",
- flag_value=False, show_default=True, default=True)
- @click.option("--enable-llcbeta", "test_llcbeta",
- help="Enable LLCBETA tests", is_flag=True)
- @click.option("--disable-lto", "test_lto",
- help="Disable use of link-time optimization",
- flag_value=False, show_default=True, default=True)
- @click.option("--small", "test_small",
- help="Use smaller test inputs and disable large tests",
- is_flag=True)
- @click.option("--large", "test_large",
- help="Use larger test inputs", is_flag=True)
- @click.option("--spec-with-ref", "test_spec_ref",
- help="Use reference test inputs for SPEC. "
- "This is currently experimental", is_flag=True)
- @click.option("--benchmarking-only", "test_benchmarking_only",
- help="Benchmarking-only mode", is_flag=True)
- @click.option("--only-test", "only_test", metavar="PATH",
- help="Only run tests under PATH",
- type=click.UNPROCESSED, default=None)
- @click.option("--include-test-examples",
- help="Include test module examples",
- is_flag=True)
- # Test Execution
- @click.option("-j", "--threads", "threads",
- help="Number of testing threads",
- type=int, default=1, metavar="N")
- @click.option("--build-threads", "build_threads",
- help="Number of compilation threads",
- type=int, default=0, metavar="N")
- @click.option("--use-perf", type=click.UNPROCESSED, default=None,
- help="Use perf to obtain high accuracy timing")
- @click.option("--rerun", help="Rerun tests that have regressed.",
- is_flag=True)
- @click.option("--remote", is_flag=True,
- help="Execute remotely, see "
- "--remote-{host,port,user,client}")
- @click.option("--remote-host", default="localhost", metavar="HOST",
- help="Set remote execution host", type=click.UNPROCESSED)
- @click.option("--remote-port", type=int, default=None,
- metavar="PORT", help="Set remote execution port")
- @click.option("--remote-user", help="Set remote execution user",
- type=click.UNPROCESSED, default=None, metavar="USER",)
- @click.option("--remote-client", type=click.UNPROCESSED, default="ssh",
- help="Set remote execution client", metavar="RSH")
- @click.option("--use-ios-simulator", "ios_simulator_sdk",
- help=("Execute using an iOS simulator SDK (using "
- "environment overrides)"),
- type=click.UNPROCESSED, default=None, metavar="SDKPATH")
- @click.option("--use-isolation", "use_isolation",
- help=("Execute using a sandboxing profile to limit "
- "OS access (e.g., to the network or "
- "non-test directories)"),
- is_flag=True)
- @click.option("--qemu-user-mode", "qemu_user_mode",
- help=("Enable qemu user mode emulation using this "
- "qemu executable"),
- type=click.UNPROCESSED, default=None)
- @click.option("--qemu-flag", "qemu_flags",
- help="Additional flags to pass to qemu", multiple=True,
- type=click.UNPROCESSED, metavar="FLAG")
- @click.option("--qemu-flags", "qemu_string",
- help="Additional flags to pass to qemu, space "
- "separated string. These flags are appended after "
- "*all* the individual --qemu-flag arguments.",
- type=click.UNPROCESSED, default='', metavar="FLAG")
- @click.option("--multisample", "multisample",
- help="Accumulate test data from multiple runs",
- type=int, default=None, metavar="N")
- # Output Options
- @click.option("--no-auto-name", "auto_name",
- help="Don't automatically derive submission name",
- flag_value=False, show_default=True, default=True)
- @click.option("--no-machdep-info", "use_machdep_info",
- help=("Don't put machine (instance) dependent "
- "variables with machine info"),
- flag_value=False, show_default=True, default=True)
- @click.option("--run-order", "run_order", metavar="STR",
- help="String to use to identify and order this run",
- type=click.UNPROCESSED, default=None)
- @click.option("--machine-param", "machine_parameters",
- metavar="NAME=VAL",
- help="Add 'NAME' = 'VAL' to the machine parameters",
- type=click.UNPROCESSED, multiple=True, default=[])
- @click.option("--run-param", "run_parameters",
- metavar="NAME=VAL",
- help="Add 'NAME' = 'VAL' to the run parameters",
- type=click.UNPROCESSED, multiple=True, default=[])
- @click.option("--submit", "submit_url", metavar="URLORPATH",
- help=("autosubmit the test result to the given server"
- " (or local instance)"),
- type=click.UNPROCESSED, default=[], multiple=True)
- @click.option("--commit", "commit",
- help="whether the autosubmit result should be committed",
- type=int, default=True)
- @click.option("--output", "output", metavar="PATH",
- help="write raw report data to PATH (or stdout if '-')",
- default=None)
- @click.option("-v", "--verbose", "verbose",
- help="show verbose test results", is_flag=True)
- @click.option("--exclude-stat-from-submission",
- "exclude_stat_from_submission",
- help="Do not submit the stat of this type ",
- multiple=True, type=click.Choice(KNOWN_SAMPLE_KEYS),
- default=['hash'])
- def cli_wrapper(*args, **kwargs):
- """LLVM test-suite compile and execution tests"""
- _tools_check()
- nt = NTTest()
-
- for key, value in kwargs.items():
- setattr(nt.opts, key, value)
-
- results = nt.run_test(nt.opts)
- nt.show_results_url(results)
-
-
def run_test(self, opts):
opts.cflag_string = ' '.join(opts.cflag_string)
@@ -2009,3 +1783,225 @@ def _tools_check():
status = call(["which", "tclsh"], stdout=FNULL, stderr=FNULL)
if status > 0:
raise SystemExit("""error: tclsh not available on your system.""")
+
+# FIXME: an equivalent to argparse's add_argument_group is not implemented
+# on click. Need to review it when such functionality is available.
+# https://github.com/pallets/click/issues/373
+ at click.command("nt", help=usage_info, short_help=__doc__)
+ at click.argument("label", default=platform.uname()[1], required=False,
+ type=click.UNPROCESSED)
+# Sandbox
+ at click.option("-s", "--sandbox", "sandbox_path", required=True,
+ help="parent directory to build and run tests in",
+ type=click.UNPROCESSED)
+ at click.option("--no-timestamp", "timestamp_build", flag_value=False,
+ default=True, show_default=True,
+ help="don't timestamp build directory (for testing)")
+ at click.option("--no-configure", "run_configure", flag_value=False,
+ default=True, show_default=True,
+ help="don't run configure if Makefile.config is "
+ "present (only useful with --no-timestamp)")
+# Inputs
+ at click.option("--without-llvm", is_flag=True, show_default=True,
+ help="don't use any LLVM source or build products")
+ at click.option("--llvm-src", "llvm_src_root",
+ help="path to the LLVM source tree",
+ type=click.UNPROCESSED)
+ at click.option("--llvm-obj", "llvm_obj_root", metavar="PATH",
+ help="path to the LLVM source tree",
+ type=click.UNPROCESSED)
+ at click.option("--test-suite", "test_suite_root", metavar="PATH",
+ help="path to the LLVM test-suite sources",
+ type=click.UNPROCESSED)
+ at click.option("--test-externals", "test_suite_externals",
+ show_default=True,
+ help="path to the LLVM test-suite externals",
+ default='/dev/null', metavar="PATH",
+ type=click.UNPROCESSED)
+# Test Compiler
+ at click.option("--cc", "cc_under_test", metavar="CC",
+ help="path to the C compiler to test",
+ type=click.UNPROCESSED)
+ at click.option("--cxx", "cxx_under_test", metavar="CXX",
+ help="path to the C++ compiler to test",
+ type=click.UNPROCESSED)
+ at click.option("--cc-reference",
+ help="path to the reference C compiler",
+ type=click.UNPROCESSED)
+ at click.option("--cxx-reference",
+ help="path to the reference C++ compiler",
+ type=click.UNPROCESSED)
+# Test Options
+ at click.option("--arch", help="Set -arch in TARGET_FLAGS")
+ at click.option("--llvm-arch",
+ help="Set the ARCH value used in the makefiles to",
+ type=click.UNPROCESSED)
+ at click.option("--make-param", "make_parameters", multiple=True,
+ help="Add 'NAME' = 'VAL' to the makefile parameters",
+ type=click.UNPROCESSED)
+ at click.option("--isysroot", "isysroot", metavar="PATH",
+ help="Set -isysroot in TARGET_FLAGS",
+ type=click.UNPROCESSED)
+ at click.option("--liblto-path", metavar="PATH",
+ help="Specify the path to the libLTO library",
+ type=click.UNPROCESSED)
+ at click.option("--mcpu", metavar="CPU",
+ help="Set -mcpu in TARGET_LLCFLAGS",
+ type=click.UNPROCESSED)
+ at click.option("--relocation-model", metavar="MODEL",
+ help="Set -relocation-model in TARGET_LLCFLAGS",
+ type=click.UNPROCESSED)
+ at click.option("--disable-fp-elim", is_flag=True,
+ help="Set -disable-fp-elim in TARGET_LLCFLAGS")
+ at click.option("--optimize-option", show_default=True,
+ help="Set optimization level for {LLC_,LLI_,}OPTFLAGS",
+ type=click.Choice(['-O0', '-O1', '-O2',
+ '-O3', '-Os', '-Oz']),
+ default='-O3')
+ at click.option("--cflag", "cflags", multiple=True,
+ help="Additional flags to set in TARGET_FLAGS",
+ type=click.UNPROCESSED, default=[], metavar="FLAG")
+ at click.option("--cflags", "cflag_string", multiple=True,
+ help="Additional flags to set in TARGET_FLAGS, space "
+ "separated string. These flags are appended after "
+ "*all* the individual --cflag arguments.",
+ type=click.UNPROCESSED, default='', metavar="FLAG")
+ at click.option("--mllvm", multiple=True,
+ help="Add -mllvm FLAG to TARGET_FLAGS",
+ type=click.UNPROCESSED, default=[], metavar="FLAG")
+ at click.option("--spec-with-pgo", is_flag=True, help="Use PGO with SPEC")
+ at click.option("--build-mode", metavar="NAME", show_default=True,
+ default='Release+Asserts',
+ help="Select the LLVM build mode to use",
+ type=click.UNPROCESSED)
+ at click.option("--simple", "test_simple", is_flag=True,
+ help="Use TEST=simple instead of TEST=nightly")
+ at click.option("--test-style", type=click.Choice(['nightly', 'simple']),
+ default='simple', help="Set the test style to run")
+ at click.option("--test-time-stat", type=click.Choice(['user', 'real']),
+ default='user', show_default=True,
+ help="Set the test timing statistic to gather")
+ at click.option("--disable-cxx", "test_cxx", flag_value=False, default=True,
+ show_default=True, help="Disable C++ tests")
+ at click.option("--disable-externals", "test_externals", flag_value=False,
+ default=True, show_default=True,
+ help="Disable test suite externals (if configured)")
+ at click.option("--enable-integrated-as", "test_integrated_as", is_flag=True,
+ help="Enable TEST_INTEGRATED_AS tests")
+ at click.option("--enable-jit", "test_jit", is_flag=True,
+ help="Enable JIT tests")
+ at click.option("--disable-llc", "test_llc",
+ help="Disable LLC tests",
+ flag_value=False, show_default=True, default=True)
+ at click.option("--enable-llcbeta", "test_llcbeta",
+ help="Enable LLCBETA tests", is_flag=True)
+ at click.option("--disable-lto", "test_lto",
+ help="Disable use of link-time optimization",
+ flag_value=False, show_default=True, default=True)
+ at click.option("--small", "test_small",
+ help="Use smaller test inputs and disable large tests",
+ is_flag=True)
+ at click.option("--large", "test_large",
+ help="Use larger test inputs", is_flag=True)
+ at click.option("--spec-with-ref", "test_spec_ref",
+ help="Use reference test inputs for SPEC. "
+ "This is currently experimental", is_flag=True)
+ at click.option("--benchmarking-only", "test_benchmarking_only",
+ help="Benchmarking-only mode", is_flag=True)
+ at click.option("--only-test", "only_test", metavar="PATH",
+ help="Only run tests under PATH",
+ type=click.UNPROCESSED, default=None)
+ at click.option("--include-test-examples",
+ help="Include test module examples",
+ is_flag=True)
+# Test Execution
+ at click.option("-j", "--threads", "threads",
+ help="Number of testing threads",
+ type=int, default=1, metavar="N")
+ at click.option("--build-threads", "build_threads",
+ help="Number of compilation threads",
+ type=int, default=0, metavar="N")
+ at click.option("--use-perf", type=click.UNPROCESSED, default=None,
+ help="Use perf to obtain high accuracy timing")
+ at click.option("--rerun", help="Rerun tests that have regressed.",
+ is_flag=True)
+ at click.option("--remote", is_flag=True,
+ help="Execute remotely, see "
+ "--remote-{host,port,user,client}")
+ at click.option("--remote-host", default="localhost", metavar="HOST",
+ help="Set remote execution host", type=click.UNPROCESSED)
+ at click.option("--remote-port", type=int, default=None,
+ metavar="PORT", help="Set remote execution port")
+ at click.option("--remote-user", help="Set remote execution user",
+ type=click.UNPROCESSED, default=None, metavar="USER",)
+ at click.option("--remote-client", type=click.UNPROCESSED, default="ssh",
+ help="Set remote execution client", metavar="RSH")
+ at click.option("--use-ios-simulator", "ios_simulator_sdk",
+ help=("Execute using an iOS simulator SDK (using "
+ "environment overrides)"),
+ type=click.UNPROCESSED, default=None, metavar="SDKPATH")
+ at click.option("--use-isolation", "use_isolation",
+ help=("Execute using a sandboxing profile to limit "
+ "OS access (e.g., to the network or "
+ "non-test directories)"),
+ is_flag=True)
+ at click.option("--qemu-user-mode", "qemu_user_mode",
+ help=("Enable qemu user mode emulation using this "
+ "qemu executable"),
+ type=click.UNPROCESSED, default=None)
+ at click.option("--qemu-flag", "qemu_flags",
+ help="Additional flags to pass to qemu", multiple=True,
+ type=click.UNPROCESSED, metavar="FLAG")
+ at click.option("--qemu-flags", "qemu_string",
+ help="Additional flags to pass to qemu, space "
+ "separated string. These flags are appended after "
+ "*all* the individual --qemu-flag arguments.",
+ type=click.UNPROCESSED, default='', metavar="FLAG")
+ at click.option("--multisample", "multisample",
+ help="Accumulate test data from multiple runs",
+ type=int, default=None, metavar="N")
+# Output Options
+ at click.option("--no-auto-name", "auto_name",
+ help="Don't automatically derive submission name",
+ flag_value=False, show_default=True, default=True)
+ at click.option("--no-machdep-info", "use_machdep_info",
+ help=("Don't put machine (instance) dependent "
+ "variables with machine info"),
+ flag_value=False, show_default=True, default=True)
+ at click.option("--run-order", "run_order", metavar="STR",
+ help="String to use to identify and order this run",
+ type=click.UNPROCESSED, default=None)
+ at click.option("--machine-param", "machine_parameters",
+ metavar="NAME=VAL",
+ help="Add 'NAME' = 'VAL' to the machine parameters",
+ type=click.UNPROCESSED, multiple=True, default=[])
+ at click.option("--run-param", "run_parameters",
+ metavar="NAME=VAL",
+ help="Add 'NAME' = 'VAL' to the run parameters",
+ type=click.UNPROCESSED, multiple=True, default=[])
+ at click.option("--submit", "submit_url", metavar="URLORPATH",
+ help=("autosubmit the test result to the given server"
+ " (or local instance)"),
+ type=click.UNPROCESSED, default=[], multiple=True)
+ at click.option("--commit", "commit",
+ help="whether the autosubmit result should be committed",
+ type=int, default=True)
+ at click.option("--output", "output", metavar="PATH",
+ help="write raw report data to PATH (or stdout if '-')",
+ default=None)
+ at click.option("-v", "--verbose", "verbose",
+ help="show verbose test results", is_flag=True)
+ at click.option("--exclude-stat-from-submission",
+ "exclude_stat_from_submission",
+ help="Do not submit the stat of this type ",
+ multiple=True, type=click.Choice(KNOWN_SAMPLE_KEYS),
+ default=['hash'])
+def cli_action(*args, **kwargs):
+ _tools_check()
+ nt = NTTest()
+
+ for key, value in kwargs.items():
+ setattr(nt.opts, key, value)
+
+ results = nt.run_test(nt.opts)
+ nt.show_results_url(results)
Modified: lnt/trunk/lnt/tests/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/tests/test_suite.py?rev=307803&r1=307802&r2=307803&view=diff
==============================================================================
--- lnt/trunk/lnt/tests/test_suite.py (original)
+++ lnt/trunk/lnt/tests/test_suite.py Wed Jul 12 09:14:56 2017
@@ -175,151 +175,6 @@ class TestSuiteTest(BuiltinTest):
self.compiled = False
self.trained = False
- @staticmethod
- @click.command("test-suite")
- @click.argument("label", default=platform.uname()[1], required=False,
- type=click.UNPROCESSED)
- # Sandbox options
- @click.option("-S", "--sandbox", "sandbox_path", required=True,
- help="Parent directory to build and run tests in",
- type=click.UNPROCESSED, metavar="PATH")
- @click.option("--no-timestamp", "timestamp_build",
- flag_value=False, default=True,
- help="Don't timestamp build directory (for testing)")
- @click.option("--no-configure", "run_configure",
- flag_value=False, default=True,
- help="Don't run CMake if CMakeCache.txt is present"
- " (only useful with --no-timestamp")
- # Inputs
- @click.option("--test-suite", "test_suite_root",
- type=click.UNPROCESSED, metavar="PATH",
- help="Path to the LLVM test-suite sources")
- @click.option("--test-externals", "test_suite_externals",
- type=click.UNPROCESSED, metavar="PATH",
- help="Path to the LLVM test-suite externals")
- @click.option("--cmake-define", "cmake_defines",
- multiple=True,
- help="Defines to pass to cmake. These do not require the "
- "-D prefix and can be given multiple times. e.g.: "
- "--cmake-define A=B => -DA=B")
- @click.option("-C", "--cmake-cache", "cmake_cache", multiple=True,
- default=[],
- help="Use one of the test-suite's cmake configurations."
- " Ex: Release, Debug")
- # Test compiler
- @click.option("--cc", "cc", metavar="CC", type=click.UNPROCESSED,
- default=None,
- help="Path to the C compiler to test")
- @click.option("--cxx", "cxx", metavar="CXX", type=click.UNPROCESSED,
- default=None,
- help="Path to the C++ compiler to test (inferred from"
- " --cc where possible")
- @click.option("--cppflags", "cppflags", type=click.UNPROCESSED,
- multiple=True, default=[],
- help="Extra flags to pass the compiler in C or C++ mode. "
- "Can be given multiple times")
- @click.option("--cflags", "--cflag", "cflags", type=click.UNPROCESSED,
- multiple=True, default=[],
- help="Extra CFLAGS to pass to the compiler. Can be "
- "given multiple times")
- @click.option("--cxxflags", "cxxflags", type=click.UNPROCESSED,
- multiple=True, default=[],
- help="Extra CXXFLAGS to pass to the compiler. Can be "
- "given multiple times")
- # Test selection
- @click.option("--test-size", "test_size",
- type=click.Choice(['small', 'regular', 'large']),
- default='regular', help="The size of test inputs to use")
- @click.option("--benchmarking-only", "benchmarking_only", is_flag=True,
- help="Benchmarking-only mode. Disable unit tests and "
- "other flaky or short-running tests")
- @click.option("--only-test", "only_test", metavar="PATH",
- type=click.UNPROCESSED, default=None,
- help="Only run tests under PATH")
- # Test Execution
- @click.option("--only-compile", "only_compile",
- help="Don't run the tests, just compile them.", is_flag=True)
- @click.option("-j", "--threads", "threads",
- help="Number of testing (and optionally build) "
- "threads", type=int, default=1, metavar="N")
- @click.option("--build-threads", "build_threads",
- help="Number of compilation threads, defaults to --threads",
- type=int, default=0, metavar="N")
- @click.option("--use-perf", "use_perf",
- help="Use Linux perf for high accuracy timing, profile "
- "information or both",
- type=click.Choice(['none', 'time', 'profile', 'all']),
- default='none')
- @click.option("--perf-events", "perf_events",
- help=("Define which linux perf events to measure"),
- type=click.UNPROCESSED, default=None)
- @click.option("--run-under", "run_under", default="",
- help="Wrapper to run tests under", type=click.UNPROCESSED)
- @click.option("--exec-multisample", "exec_multisample",
- help="Accumulate execution test data from multiple runs",
- type=int, default=1, metavar="N")
- @click.option("--compile-multisample", "compile_multisample",
- help="Accumulate compile test data from multiple runs",
- type=int, default=1, metavar="N")
- @click.option("-d", "--diagnose", "diagnose",
- help="Produce a diagnostic report for a particular "
- "test, this will not run all the tests. Must be"
- " used in conjunction with --only-test.",
- is_flag=True, default=False,)
- @click.option("--pgo", "pgo",
- help="Run the test-suite in training mode first and"
- " collect PGO data, then rerun with that training "
- "data.",
- is_flag=True, default=False,)
- # Output Options
- @click.option("--no-auto-name", "auto_name",
- help="Don't automatically derive submission name",
- flag_value=False, default=True)
- @click.option("--run-order", "run_order", metavar="STR",
- help="String to use to identify and order this run")
- @click.option("--submit", "submit_url", metavar="URLORPATH",
- help="autosubmit the test result to the given server"
- " (or local instance)",
- type=click.UNPROCESSED, default=None)
- @click.option("--commit", "commit",
- help="whether the autosubmit result should be committed",
- type=int, default=True)
- @click.option("--succinct-compile-output", "succinct",
- help="run Make without VERBOSE=1", is_flag=True)
- @click.option("-v", "--verbose", "verbose", is_flag=True, default=False,
- help="show verbose test results")
- @click.option("--exclude-stat-from-submission",
- "exclude_stat_from_submission",
- help="Do not submit the stat of this type",
- multiple=True, default=[],
- type=click.Choice(KNOWN_SAMPLE_KEYS))
- @click.option("--single-result", "single_result",
- help="only execute this single test and apply "
- "--single-result-predicate to calculate the exit "
- "status")
- @click.option("--single-result-predicate", "single_result_predicate",
- help="the predicate to apply to calculate the exit "
- "status (with --single-result)", default="status")
- # Test tools
- @click.option("--use-cmake", "cmake", metavar="PATH",
- type=click.UNPROCESSED, default="cmake",
- help="Path to CMake [cmake]")
- @click.option("--use-make", "make", metavar="PATH",
- type=click.UNPROCESSED, default="make",
- help="Path to Make [make]")
- @click.option("--use-lit", "lit", metavar="PATH", type=click.UNPROCESSED,
- default="llvm-lit",
- help="Path to the LIT test runner [llvm-lit]")
- def cli_wrapper(*args, **kwargs):
- """LLVM test-suite"""
- test_suite = TestSuiteTest()
-
- for key, value in kwargs.items():
- setattr(test_suite.opts, key, value)
-
- results = test_suite.run_test(test_suite.opts)
- test_suite.show_results_url(results)
-
def run_test(self, opts):
if self.opts.cc is not None:
@@ -1094,3 +949,146 @@ class TestSuiteTest(BuiltinTest):
return report_path
return DontSubmitResults()
+
+ at click.command("test-suite", short_help=__doc__)
+ at click.argument("label", default=platform.uname()[1], required=False,
+ type=click.UNPROCESSED)
+# Sandbox options
+ at click.option("-S", "--sandbox", "sandbox_path", required=True,
+ help="Parent directory to build and run tests in",
+ type=click.UNPROCESSED, metavar="PATH")
+ at click.option("--no-timestamp", "timestamp_build",
+ flag_value=False, default=True,
+ help="Don't timestamp build directory (for testing)")
+ at click.option("--no-configure", "run_configure",
+ flag_value=False, default=True,
+ help="Don't run CMake if CMakeCache.txt is present"
+ " (only useful with --no-timestamp")
+# Inputs
+ at click.option("--test-suite", "test_suite_root",
+ type=click.UNPROCESSED, metavar="PATH",
+ help="Path to the LLVM test-suite sources")
+ at click.option("--test-externals", "test_suite_externals",
+ type=click.UNPROCESSED, metavar="PATH",
+ help="Path to the LLVM test-suite externals")
+ at click.option("--cmake-define", "cmake_defines",
+ multiple=True,
+ help="Defines to pass to cmake. These do not require the "
+ "-D prefix and can be given multiple times. e.g.: "
+ "--cmake-define A=B => -DA=B")
+ at click.option("-C", "--cmake-cache", "cmake_cache", multiple=True,
+ default=[],
+ help="Use one of the test-suite's cmake configurations."
+ " Ex: Release, Debug")
+# Test compiler
+ at click.option("--cc", "cc", metavar="CC", type=click.UNPROCESSED,
+ default=None,
+ help="Path to the C compiler to test")
+ at click.option("--cxx", "cxx", metavar="CXX", type=click.UNPROCESSED,
+ default=None,
+ help="Path to the C++ compiler to test (inferred from"
+ " --cc where possible")
+ at click.option("--cppflags", "cppflags", type=click.UNPROCESSED,
+ multiple=True, default=[],
+ help="Extra flags to pass the compiler in C or C++ mode. "
+ "Can be given multiple times")
+ at click.option("--cflags", "--cflag", "cflags", type=click.UNPROCESSED,
+ multiple=True, default=[],
+ help="Extra CFLAGS to pass to the compiler. Can be "
+ "given multiple times")
+ at click.option("--cxxflags", "cxxflags", type=click.UNPROCESSED,
+ multiple=True, default=[],
+ help="Extra CXXFLAGS to pass to the compiler. Can be "
+ "given multiple times")
+# Test selection
+ at click.option("--test-size", "test_size",
+ type=click.Choice(['small', 'regular', 'large']),
+ default='regular', help="The size of test inputs to use")
+ at click.option("--benchmarking-only", "benchmarking_only", is_flag=True,
+ help="Benchmarking-only mode. Disable unit tests and "
+ "other flaky or short-running tests")
+ at click.option("--only-test", "only_test", metavar="PATH",
+ type=click.UNPROCESSED, default=None,
+ help="Only run tests under PATH")
+# Test Execution
+ at click.option("--only-compile", "only_compile",
+ help="Don't run the tests, just compile them.", is_flag=True)
+ at click.option("-j", "--threads", "threads",
+ help="Number of testing (and optionally build) "
+ "threads", type=int, default=1, metavar="N")
+ at click.option("--build-threads", "build_threads",
+ help="Number of compilation threads, defaults to --threads",
+ type=int, default=0, metavar="N")
+ at click.option("--use-perf", "use_perf",
+ help="Use Linux perf for high accuracy timing, profile "
+ "information or both",
+ type=click.Choice(['none', 'time', 'profile', 'all']),
+ default='none')
+ at click.option("--perf-events", "perf_events",
+ help=("Define which linux perf events to measure"),
+ type=click.UNPROCESSED, default=None)
+ at click.option("--run-under", "run_under", default="",
+ help="Wrapper to run tests under", type=click.UNPROCESSED)
+ at click.option("--exec-multisample", "exec_multisample",
+ help="Accumulate execution test data from multiple runs",
+ type=int, default=1, metavar="N")
+ at click.option("--compile-multisample", "compile_multisample",
+ help="Accumulate compile test data from multiple runs",
+ type=int, default=1, metavar="N")
+ at click.option("-d", "--diagnose", "diagnose",
+ help="Produce a diagnostic report for a particular "
+ "test, this will not run all the tests. Must be"
+ " used in conjunction with --only-test.",
+ is_flag=True, default=False,)
+ at click.option("--pgo", "pgo",
+ help="Run the test-suite in training mode first and"
+ " collect PGO data, then rerun with that training "
+ "data.",
+ is_flag=True, default=False,)
+# Output Options
+ at click.option("--no-auto-name", "auto_name",
+ help="Don't automatically derive submission name",
+ flag_value=False, default=True)
+ at click.option("--run-order", "run_order", metavar="STR",
+ help="String to use to identify and order this run")
+ at click.option("--submit", "submit_url", metavar="URLORPATH",
+ help="autosubmit the test result to the given server"
+ " (or local instance)",
+ type=click.UNPROCESSED, default=None)
+ at click.option("--commit", "commit",
+ help="whether the autosubmit result should be committed",
+ type=int, default=True)
+ at click.option("--succinct-compile-output", "succinct",
+ help="run Make without VERBOSE=1", is_flag=True)
+ at click.option("-v", "--verbose", "verbose", is_flag=True, default=False,
+ help="show verbose test results")
+ at click.option("--exclude-stat-from-submission",
+ "exclude_stat_from_submission",
+ help="Do not submit the stat of this type",
+ multiple=True, default=[],
+ type=click.Choice(KNOWN_SAMPLE_KEYS))
+ at click.option("--single-result", "single_result",
+ help="only execute this single test and apply "
+ "--single-result-predicate to calculate the exit "
+ "status")
+ at click.option("--single-result-predicate", "single_result_predicate",
+ help="the predicate to apply to calculate the exit "
+ "status (with --single-result)", default="status")
+# Test tools
+ at click.option("--use-cmake", "cmake", metavar="PATH",
+ type=click.UNPROCESSED, default="cmake",
+ help="Path to CMake [cmake]")
+ at click.option("--use-make", "make", metavar="PATH",
+ type=click.UNPROCESSED, default="make",
+ help="Path to Make [make]")
+ at click.option("--use-lit", "lit", metavar="PATH", type=click.UNPROCESSED,
+ default="llvm-lit",
+ help="Path to the LIT test runner [llvm-lit]")
+def cli_action(*args, **kwargs):
+ test_suite = TestSuiteTest()
+
+ for key, value in kwargs.items():
+ setattr(test_suite.opts, key, value)
+
+ results = test_suite.run_test(test_suite.opts)
+ test_suite.show_results_url(results)
More information about the llvm-commits
mailing list