[llvm] 1e2d509 - Add lit config for dir with standalone tests
Thomas Preud'homme via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 17 02:38:55 PST 2021
Author: Thomas Preud'homme
Date: 2021-02-17T10:38:58Z
New Revision: 1e2d50936a3bff2c8065a99d06f37b51185fbd81
URL: https://github.com/llvm/llvm-project/commit/1e2d50936a3bff2c8065a99d06f37b51185fbd81
DIFF: https://github.com/llvm/llvm-project/commit/1e2d50936a3bff2c8065a99d06f37b51185fbd81.diff
LOG: Add lit config for dir with standalone tests
Some test systems do not use lit for test discovery but only for its
substitution and test selection because they use another way of managing
test collections, e.g. CTest. This forces those tests to be invoked with
lit --no-indirectly-run-check. When a mix of lit version is in use, it
requires to detect the availability of that option.
This commit provides a new config option standalone_tests to signal a
directory made of tests meant to run as standalone. When this option is
set, lit skips test discovery and the indirectly run check. It also adds
the missing documentation for --no-indirectly-run-check.
Reviewed By: jdenny
Differential Revision: https://reviews.llvm.org/D94766
Added:
llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg
llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt
llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg
llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt
llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg
llvm/utils/lit/tests/Inputs/standalone-tests/true.txt
Modified:
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/TestingConfig.py
llvm/utils/lit/lit/discovery.py
llvm/utils/lit/tests/discovery.py
Removed:
################################################################################
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index f4d9d9d7e6a1..3820cc446c3c 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -154,6 +154,11 @@ EXECUTION OPTIONS
suite take the most time to execute. Note that this option is most useful
with ``-j 1``.
+.. option:: --no-indirectly-run-check
+
+ Do not error if a test would not be run if the user had specified the
+ containing directory instead of naming the test directly.
+
.. _selection-options:
SELECTION OPTIONS
@@ -372,6 +377,11 @@ executed, two important global variables are predefined:
**environment** A dictionary representing the environment to use when executing
tests in the suite.
+ **standalone_tests** When true, mark a directory with tests expected to be run
+ standalone. Test discovery is disabled for that directory and
+ *--no-indirectly-run-check* is in effect. *lit.suffixes* and *lit.excludes*
+ must be empty when this variable is true.
+
**suffixes** For **lit** test formats which scan directories for tests, this
variable is a list of suffixes to identify test files. Used by: *ShTest*.
diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py
index 38d05066a2b0..d40ee37636f3 100644
--- a/llvm/utils/lit/lit/TestingConfig.py
+++ b/llvm/utils/lit/lit/TestingConfig.py
@@ -60,7 +60,8 @@ def fromdefaults(litConfig):
test_source_root = None,
excludes = [],
available_features = available_features,
- pipefail = True)
+ pipefail = True,
+ standalone_tests = False)
def load_from_path(self, path, litConfig):
"""
@@ -105,7 +106,8 @@ def __init__(self, parent, name, suffixes, test_format,
environment, substitutions, unsupported,
test_exec_root, test_source_root, excludes,
available_features, pipefail, limit_to_features = [],
- is_early = False, parallelism_group = None):
+ is_early = False, parallelism_group = None,
+ standalone_tests = False):
self.parent = parent
self.name = str(name)
self.suffixes = set(suffixes)
@@ -118,6 +120,7 @@ def __init__(self, parent, name, suffixes, test_format,
self.excludes = set(excludes)
self.available_features = set(available_features)
self.pipefail = pipefail
+ self.standalone_tests = standalone_tests
# This list is used by TestRunner.py to restrict running only tests that
# require one of the features in this list if this list is non-empty.
# Configurations can set this list to restrict the set of tests to run.
diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py
index 2f027a5b03fe..a185ae676d14 100644
--- a/llvm/utils/lit/lit/discovery.py
+++ b/llvm/utils/lit/lit/discovery.py
@@ -160,8 +160,13 @@ def getTestsInSuite(ts, path_in_suite, litConfig,
# tests which are not executed. The check adds some performance
# overhead which might be important if a large number of tests
# are being run directly.
- # --no-indirectly-run-check: skips this check.
- if indirectlyRunCheck and lc.test_format is not None:
+ # This check can be disabled by using --no-indirectly-run-check or
+ # setting the standalone_tests variable in the suite's configuration.
+ if (
+ indirectlyRunCheck
+ and lc.test_format is not None
+ and not lc.standalone_tests
+ ):
found = False
for res in lc.test_format.getTestsInDirectory(ts, test_dir_in_suite,
litConfig, lc):
@@ -171,6 +176,7 @@ def getTestsInSuite(ts, path_in_suite, litConfig,
if not found:
litConfig.error(
'%r would not be run indirectly: change name or LIT config'
+ '(e.g. suffixes or standalone_tests variables)'
% test.getFullName())
yield test
@@ -180,6 +186,15 @@ def getTestsInSuite(ts, path_in_suite, litConfig,
# local configuration.
lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
+ # Directory contains tests to be run standalone. Do not try to discover.
+ if lc.standalone_tests:
+ if lc.suffixes or lc.excludes:
+ litConfig.warning(
+ 'standalone_tests set in LIT config but suffixes or excludes'
+ ' are also set'
+ )
+ return
+
# Search for tests.
if lc.test_format is not None:
for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg
new file mode 100644
index 000000000000..37d96ddaebfe
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg
@@ -0,0 +1,5 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.excludes = ['.test']
+config.standalone_tests = True
diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt
new file mode 100644
index 000000000000..b80b60b7a279
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg
new file mode 100644
index 000000000000..41434c9855b3
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg
@@ -0,0 +1,5 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.suffixes = ['.txt']
+config.standalone_tests = True
diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt
new file mode 100644
index 000000000000..b80b60b7a279
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg
new file mode 100644
index 000000000000..12ed7b0ce067
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg
@@ -0,0 +1,4 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.standalone_tests = True
diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt
new file mode 100644
index 000000000000..b80b60b7a279
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/discovery.py b/llvm/utils/lit/tests/discovery.py
index cb04edaab04c..39743bfd7c58 100644
--- a/llvm/utils/lit/tests/discovery.py
+++ b/llvm/utils/lit/tests/discovery.py
@@ -148,6 +148,36 @@
# RUN: %{lit} \
# RUN: %{inputs}/discovery/test.not-txt -j 1 --no-indirectly-run-check
+# Check that a standalone test with no suffixes set is run without any errors.
+#
+# RUN: %{lit} %{inputs}/standalone-tests/true.txt -j 1 > %t.out
+# RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s
+#
+# CHECK-STANDALONE: PASS: Standalone tests :: true.txt
+
+# Check that an error is produced if suffixes variable is set for a suite with
+# standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes -j 1 2> %t.err
+# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes
+
+# Check that an error is produced if excludes variable is set for a suite with
+# standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes -j 1 2> %t.err
+# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes
+
+# Check that no discovery is done for testsuite with standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests -j 1 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)
+
# Check that we don't recurse infinitely when loading an site specific test
# suite located inside the test source root.
#
More information about the llvm-commits
mailing list