[llvm] d8f2bff - [lit] Better/earlier errors when no tests are executed

[Julian Lettner via llvm-commits]

Author: Julian Lettner
Date: 2019-11-04T10:16:24-08:00
New Revision: d8f2bff75126c6dde694ad245f9807fa12ad5630

URL: https://github.com/llvm/llvm-project/commit/d8f2bff75126c6dde694ad245f9807fa12ad5630
DIFF: https://github.com/llvm/llvm-project/commit/d8f2bff75126c6dde694ad245f9807fa12ad5630.diff

LOG: [lit] Better/earlier errors when no tests are executed

Fail early, when we discover no tests at all, or filter out all of them.

Added: 
    

Modified: 
    llvm/utils/lit/lit/cl_arguments.py
    llvm/utils/lit/lit/main.py
    llvm/utils/lit/lit/run.py
    llvm/utils/lit/tests/selecting.py

Removed: 
    


################################################################################
diff  --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 7b15041d1905..c17f6c7824e1 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -5,6 +5,7 @@
 
 import lit.util
 
+# TODO(yln): scan dest, to see if is redundant
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('test_paths',

diff  --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 337d154a560d..af36b07f1fef 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -42,6 +42,9 @@ def main(builtin_params = {}):
         echo_all_commands = opts.echoAllCommands)
 
     tests = lit.discovery.find_tests_for_inputs(litConfig, opts.test_paths)
+    if not tests:
+        sys.stderr.write('Did not disover any tests for provided path(s).\n')
+        sys.exit(2)
 
     # Command line overrides configuration for maxIndividualTestTime.
     if opts.maxIndividualTestTime is not None:  # `not None` is important (default: 0)
@@ -62,12 +65,20 @@ def main(builtin_params = {}):
 
     if opts.filter:
         tests = [t for t in tests if opts.filter.search(t.getFullName())]
+        if not tests:
+            sys.stderr.write('Filter did not match any tests '
+                             '(of %d discovered).\n' % numTotalTests)
+            sys.exit(2)
 
     determine_order(tests, opts.order)
 
     if opts.shard:
         (run, shards) = opts.shard
         tests = filter_by_shard(tests, run, shards, litConfig)
+        if not tests:
+            sys.stderr.write('Shard does not contain any tests.  Consider '
+                             'decreasing the shard count.\n')
+            sys.exit(0)
 
     if opts.max_tests:
         tests = tests[:opts.max_tests]
@@ -84,7 +95,7 @@ def main(builtin_params = {}):
         write_test_results_xunit(tests, opts)
 
     if litConfig.numErrors:
-        sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
+        sys.stderr.write('\n%d error(s) in tests.\n' % litConfig.numErrors)
         sys.exit(2)
 
     if litConfig.numWarnings:

diff  --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py
index 4d16399fec47..11992c8073d2 100644
--- a/llvm/utils/lit/lit/run.py
+++ b/llvm/utils/lit/lit/run.py
@@ -13,7 +13,7 @@ def acquire(self): pass
     def release(self): pass
 
 def create_run(tests, lit_config, workers, progress_callback, timeout=None):
-    # TODO(yln) assert workers > 0
+    assert workers > 0
     if workers == 1:
         return SerialRun(tests, lit_config, progress_callback, timeout)
     return ParallelRun(tests, lit_config, progress_callback, timeout, workers)
@@ -45,9 +45,6 @@ def execute(self):
         computed. Tests which were not actually executed (for any reason) will
         be given an UNRESOLVED result.
         """
-        if not self.tests:
-            return 0.0
-
         self.failure_count = 0
         self.hit_max_failures = False
 

diff  --git a/llvm/utils/lit/tests/selecting.py b/llvm/utils/lit/tests/selecting.py
index 0921cdd31ac9..cfbb9d19b6dc 100644
--- a/llvm/utils/lit/tests/selecting.py
+++ b/llvm/utils/lit/tests/selecting.py
@@ -1,6 +1,18 @@
 # RUN: %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-BASIC %s
 # CHECK-BASIC: Testing: 5 tests
 
+
+# Check that we exit with an error if we do not discover any tests.
+#
+# RUN: not %{lit} %{inputs}/nonexistent 2>&1 | FileCheck --check-prefix=CHECK-BAD-PATH %s
+# CHECK-BAD-PATH: Did not disover any tests for provided path(s).
+
+# Check that we exit with an error if we filter out all tests.
+#
+# RUN: not %{lit} --filter 'nonexistent' %{inputs}/discovery 2>&1 | FileCheck --check-prefix=CHECK-BAD-FILTER %s
+# CHECK-BAD-FILTER: Filter did not match any tests (of 5 discovered).
+
+
 # Check that regex-filtering works, is case-insensitive, and can be configured via env var.
 #
 # RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
@@ -8,6 +20,7 @@
 # RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
 # CHECK-FILTER: Testing: 2 of 5 tests
 
+
 # Check that maximum counts work
 #
 # RUN: %{lit} --max-tests 3 %{inputs}/discovery | FileCheck --check-prefix=CHECK-MAX %s
@@ -68,15 +81,13 @@
 #
 # RUN: %{lit} --num-shards 100 --run-shard 6 %{inputs}/discovery >%t.out 2>%t.err
 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR2 < %t.err %s
-# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT2 < %t.out %s
 # CHECK-SHARD-BIG-ERR2: note: Selecting shard 6/100 = size 0/5 = tests #(100*k)+6 = []
-# CHECK-SHARD-BIG-OUT2: Testing: 0 of 5 tests
+# CHECK-SHARD-BIG-ERR2: Shard does not contain any tests.  Consider decreasing the shard count.
 #
 # RUN: %{lit} --num-shards 100 --run-shard 50 %{inputs}/discovery >%t.out 2>%t.err
 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR3 < %t.err %s
-# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT3 < %t.out %s
 # CHECK-SHARD-BIG-ERR3: note: Selecting shard 50/100 = size 0/5 = tests #(100*k)+50 = []
-# CHECK-SHARD-BIG-OUT3: Testing: 0 of 5 tests
+# CHECK-SHARD-BIG-ERR3: Shard does not contain any tests.  Consider decreasing the shard count.
 
 
 # Check that range constraints are enforced