[llvm] r282452 - [lit] Add a --max-failures option.
Vedant Kumar via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 26 22:33:22 PDT 2016
Thank you, this is awesome!
vedant
> On Sep 26, 2016, at 4:38 PM, Daniel Dunbar via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>
> Author: ddunbar
> Date: Mon Sep 26 18:38:23 2016
> New Revision: 282452
>
> URL: http://llvm.org/viewvc/llvm-project?rev=282452&view=rev
> Log:
> [lit] Add a --max-failures option.
>
> - This is primarily useful as a "fail fast" mode for lit, where it will stop
> running tests after the first failure.
>
> - Patch by Max Moiseev.
>
> Added:
> llvm/trunk/utils/lit/tests/max-failures.py
> Modified:
> llvm/trunk/utils/lit/lit/LitConfig.py
> llvm/trunk/utils/lit/lit/main.py
> llvm/trunk/utils/lit/lit/run.py
>
> Modified: llvm/trunk/utils/lit/lit/LitConfig.py
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/LitConfig.py?rev=282452&r1=282451&r2=282452&view=diff
> ==============================================================================
> --- llvm/trunk/utils/lit/lit/LitConfig.py (original)
> +++ llvm/trunk/utils/lit/lit/LitConfig.py Mon Sep 26 18:38:23 2016
> @@ -23,7 +23,8 @@ class LitConfig(object):
> useValgrind, valgrindLeakCheck, valgrindArgs,
> noExecute, debug, isWindows,
> params, config_prefix = None,
> - maxIndividualTestTime = 0):
> + maxIndividualTestTime = 0,
> + maxFailures = None):
> # The name of the test runner.
> self.progname = progname
> # The items to add to the PATH environment variable.
> @@ -60,6 +61,7 @@ class LitConfig(object):
> self.valgrindArgs.extend(self.valgrindUserArgs)
>
> self.maxIndividualTestTime = maxIndividualTestTime
> + self.maxFailures = maxFailures
>
> @property
> def maxIndividualTestTime(self):
>
> Modified: llvm/trunk/utils/lit/lit/main.py
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/main.py?rev=282452&r1=282451&r2=282452&view=diff
> ==============================================================================
> --- llvm/trunk/utils/lit/lit/main.py (original)
> +++ llvm/trunk/utils/lit/lit/main.py Mon Sep 26 18:38:23 2016
> @@ -232,6 +232,9 @@ def main_with_tmp(builtinParameters):
> help="Maximum time to spend running a single test (in seconds)."
> "0 means no time limit. [Default: 0]",
> type=int, default=None)
> + group.add_option("", "--max-failures", dest="maxFailures",
> + help="Stop execution after the given number of failures.",
> + action="store", type=int, default=None)
> parser.add_option_group(group)
>
> group = OptionGroup(parser, "Test Selection")
> @@ -291,6 +294,9 @@ def main_with_tmp(builtinParameters):
> else:
> opts.numThreads = 1
>
> + if opts.maxFailures == 0:
> + parser.error("Setting --max-failures to 0 does not have any effect.")
> +
> inputs = args
>
> # Create the user defined parameters.
> @@ -324,7 +330,8 @@ def main_with_tmp(builtinParameters):
> isWindows = isWindows,
> params = userParams,
> config_prefix = opts.configPrefix,
> - maxIndividualTestTime = maxIndividualTestTime)
> + maxIndividualTestTime = maxIndividualTestTime,
> + maxFailures = opts.maxFailures)
>
> # Perform test discovery.
> run = lit.run.Run(litConfig,
> @@ -477,7 +484,8 @@ def main_with_tmp(builtinParameters):
> ('Expected Failing Tests', lit.Test.XFAIL),
> ('Timed Out Tests', lit.Test.TIMEOUT)):
> if (lit.Test.XFAIL == code and not opts.show_xfail) or \
> - (lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
> + (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
> + (lit.Test.UNRESOLVED == code and (opts.maxFailures is not None)):
> continue
> elts = byCode.get(code)
> if not elts:
>
> Modified: llvm/trunk/utils/lit/lit/run.py
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/run.py?rev=282452&r1=282451&r2=282452&view=diff
> ==============================================================================
> --- llvm/trunk/utils/lit/lit/run.py (original)
> +++ llvm/trunk/utils/lit/lit/run.py Mon Sep 26 18:38:23 2016
> @@ -152,6 +152,19 @@ def run_one_tester(run, provider, displa
>
> ###
>
> +def handleFailures(provider, consumer, maxFailures):
> + class _Display(object):
> + def __init__(self, display):
> + self.display = display
> + self.maxFailures = maxFailures or object()
> + self.failedCount = 0
> + def update(self, test):
> + self.display.update(test)
> + self.failedCount += (test.result.code == lit.Test.FAIL)
> + if self.failedCount == self.maxFailures:
> + provider.cancel()
> + consumer.display = _Display(consumer.display)
> +
> class Run(object):
> """
> This class represents a concrete, configured testing run.
> @@ -232,6 +245,7 @@ class Run(object):
>
> # Create the test provider.
> provider = TestProvider(queue_impl, canceled_flag)
> + handleFailures(provider, consumer, self.lit_config.maxFailures)
>
> # Queue the tests outside the main thread because we can't guarantee
> # that we can put() all the tests without blocking:
>
> Added: llvm/trunk/utils/lit/tests/max-failures.py
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/tests/max-failures.py?rev=282452&view=auto
> ==============================================================================
> --- llvm/trunk/utils/lit/tests/max-failures.py (added)
> +++ llvm/trunk/utils/lit/tests/max-failures.py Mon Sep 26 18:38:23 2016
> @@ -0,0 +1,14 @@
> +# Check the behavior of --max-failures option.
> +#
> +# RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out
> +# RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/shtest-shell >> %t.out
> +# RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/shtest-shell >> %t.out
> +# RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/shtest-shell 2>> %t.out
> +# RUN: FileCheck < %t.out %s
> +#
> +# END.
> +
> +# CHECK: Failing Tests (3)
> +# CHECK: Failing Tests (1)
> +# CHECK: Failing Tests (2)
> +# CHECK: error: Setting --max-failures to 0 does not have any effect.
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list