[llvm] 54a9b4c - [lit] Better/earlier errors for empty runs
Julian Lettner via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 12 09:14:41 PST 2019
Author: Julian Lettner
Date: 2019-11-12T09:11:36-08:00
New Revision: 54a9b4c02ff57e9847e0c501578e51db6f73d3be
URL: https://github.com/llvm/llvm-project/commit/54a9b4c02ff57e9847e0c501578e51db6f73d3be
DIFF: https://github.com/llvm/llvm-project/commit/54a9b4c02ff57e9847e0c501578e51db6f73d3be.diff
LOG: [lit] Better/earlier errors for empty runs
Fail early, when we discover no tests at all, or filter out all of them.
There is also `--allow-empty-runs` to disable test to allow workflows
like `LIT_FILTER=abc ninja check-all`. Apparently `check-all` invokes
lit multiple times if certain projects are enabled, which would produce
unwanted "empty runs". Specify via `LIT_OPTS=--allow-empty-runs`.
There are 3 causes for empty runs:
1) No tests discovered. This is always an error. Fix test suite config
or command line.
2) All tests filtered out. This is an error by default, but can be
suppressed via `--alow-empty-runs`. Should prevent accidentally
passing empty runs, but allow the workflow above.
3) The number of shards is greater than the number of tests. Currently,
this is never an error. Personally, I think we should consider
making this an error by default; if this happens, you are doing
something wrong. I added a warning but did not change the behavior,
since this warrants more discussion.
Reviewed By: atrick, jdenny
Differential Revision: https://reviews.llvm.org/D70105
Added:
Modified:
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/main.py
llvm/utils/lit/lit/run.py
llvm/utils/lit/tests/selecting.py
Removed:
################################################################################
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 7b15041d1905..cbdd0ec7b5a8 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -109,6 +109,9 @@ def parse_args():
dest="maxFailures",
help="Stop execution after the given number of failures.",
type=_positive_int)
+ execution_group.add_argument("--allow-empty-runs",
+ help="Do not fail the run if all tests are filtered out",
+ action="store_true")
selection_group = parser.add_argument_group("Test Selection")
selection_group.add_argument("--max-tests",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index a802b4509e09..0eb95b9c2ab5 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -43,6 +43,9 @@ def main(builtin_params = {}):
echo_all_commands = opts.echoAllCommands)
tests = lit.discovery.find_tests_for_inputs(litConfig, opts.test_paths)
+ if not tests:
+ sys.stderr.write('Did not disover any tests for provided path(s).\n')
+ sys.exit(2)
# Command line overrides configuration for maxIndividualTestTime.
if opts.maxIndividualTestTime is not None: # `not None` is important (default: 0)
@@ -63,12 +66,27 @@ def main(builtin_params = {}):
if opts.filter:
tests = [t for t in tests if opts.filter.search(t.getFullName())]
+ if not tests:
+ sys.stderr.write('Filter did not match any tests '
+ '(of %d discovered). ' % numTotalTests)
+ if opts.allow_empty_runs:
+ sys.stderr.write('Suppressing error because '
+ "'--allow-empty-runs' was specified.\n")
+ sys.exit(0)
+ else:
+ sys.stderr.write("Use '--allow-empty-runs' to suppress this "
+ 'error.\n')
+ sys.exit(2)
determine_order(tests, opts.order)
if opts.shard:
(run, shards) = opts.shard
tests = filter_by_shard(tests, run, shards, litConfig)
+ if not tests:
+ sys.stderr.write('Shard does not contain any tests. Consider '
+ 'decreasing the number of shards.\n')
+ sys.exit(0)
if opts.max_tests:
tests = tests[:opts.max_tests]
@@ -87,7 +105,7 @@ def main(builtin_params = {}):
write_test_results_xunit(tests, opts)
if litConfig.numErrors:
- sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
+ sys.stderr.write('\n%d error(s) in tests.\n' % litConfig.numErrors)
sys.exit(2)
if litConfig.numWarnings:
diff --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py
index 6a43f51cbf67..166b64e8a1e7 100644
--- a/llvm/utils/lit/lit/run.py
+++ b/llvm/utils/lit/lit/run.py
@@ -13,7 +13,7 @@ def acquire(self): pass
def release(self): pass
def create_run(tests, lit_config, workers, progress_callback, timeout=None):
- # TODO(yln) assert workers > 0
+ assert workers > 0
if workers == 1:
return SerialRun(tests, lit_config, progress_callback, timeout)
return ParallelRun(tests, lit_config, progress_callback, timeout, workers)
@@ -45,9 +45,6 @@ def execute(self):
computed. Tests which were not actually executed (for any reason) will
be given an UNRESOLVED result.
"""
- if not self.tests:
- return 0.0
-
self.failure_count = 0
self.hit_max_failures = False
diff --git a/llvm/utils/lit/tests/selecting.py b/llvm/utils/lit/tests/selecting.py
index 0921cdd31ac9..638c1088a26f 100644
--- a/llvm/utils/lit/tests/selecting.py
+++ b/llvm/utils/lit/tests/selecting.py
@@ -1,6 +1,21 @@
# RUN: %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-BASIC %s
# CHECK-BASIC: Testing: 5 tests
+
+# Check that we exit with an error if we do not discover any tests, even with --allow-empty-runs.
+#
+# RUN: not %{lit} %{inputs}/nonexistent 2>&1 | FileCheck --check-prefix=CHECK-BAD-PATH %s
+# RUN: not %{lit} %{inputs}/nonexistent --allow-empty-runs 2>&1 | FileCheck --check-prefix=CHECK-BAD-PATH %s
+# CHECK-BAD-PATH: Did not disover any tests for provided path(s).
+
+# Check that we exit with an error if we filter out all tests, but allow it with --allow-empty-runs.
+#
+# RUN: not %{lit} --filter 'nonexistent' %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s
+# RUN: %{lit} --filter 'nonexistent' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s
+# CHECK-BAD-FILTER: Filter did not match any tests (of 5 discovered).
+# CHECK-BAD-FILTER-ERROR: Use '--allow-empty-runs' to suppress this error.
+# CHECK-BAD-FILTER-ALLOW: Suppressing error because '--allow-empty-runs' was specified.
+
# Check that regex-filtering works, is case-insensitive, and can be configured via env var.
#
# RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
@@ -8,6 +23,7 @@
# RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
# CHECK-FILTER: Testing: 2 of 5 tests
+
# Check that maximum counts work
#
# RUN: %{lit} --max-tests 3 %{inputs}/discovery | FileCheck --check-prefix=CHECK-MAX %s
@@ -68,15 +84,13 @@
#
# RUN: %{lit} --num-shards 100 --run-shard 6 %{inputs}/discovery >%t.out 2>%t.err
# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR2 < %t.err %s
-# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT2 < %t.out %s
# CHECK-SHARD-BIG-ERR2: note: Selecting shard 6/100 = size 0/5 = tests #(100*k)+6 = []
-# CHECK-SHARD-BIG-OUT2: Testing: 0 of 5 tests
+# CHECK-SHARD-BIG-ERR2: Shard does not contain any tests. Consider decreasing the number of shards.
#
# RUN: %{lit} --num-shards 100 --run-shard 50 %{inputs}/discovery >%t.out 2>%t.err
# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR3 < %t.err %s
-# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT3 < %t.out %s
# CHECK-SHARD-BIG-ERR3: note: Selecting shard 50/100 = size 0/5 = tests #(100*k)+50 = []
-# CHECK-SHARD-BIG-OUT3: Testing: 0 of 5 tests
+# CHECK-SHARD-BIG-ERR3: Shard does not contain any tests. Consider decreasing the number of shards.
# Check that range constraints are enforced
More information about the llvm-commits
mailing list