[llvm] 2f69c82 - [llvm] [lit] Support forcing lexical test order
Michał Górny via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 27 11:50:48 PDT 2021
Author: Michał Górny
Date: 2021-08-27T20:47:11+02:00
New Revision: 2f69c82cec1ae05b4fdcef4ac48f48e9e2bad32b
URL: https://github.com/llvm/llvm-project/commit/2f69c82cec1ae05b4fdcef4ac48f48e9e2bad32b
DIFF: https://github.com/llvm/llvm-project/commit/2f69c82cec1ae05b4fdcef4ac48f48e9e2bad32b.diff
LOG: [llvm] [lit] Support forcing lexical test order
Add a new --order option to choose between available test orders:
the default "smart" order, predictable "lexical" order or "random"
order. Default to using lexical order and one job in the lit test
suite.
Differential Revision: https://reviews.llvm.org/D107695
Added:
Modified:
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/main.py
llvm/utils/lit/tests/allow-retries.py
llvm/utils/lit/tests/custom-result-category.py
llvm/utils/lit/tests/discovery.py
llvm/utils/lit/tests/googletest-discovery-failed.py
llvm/utils/lit/tests/googletest-format.py
llvm/utils/lit/tests/googletest-timeout.py
llvm/utils/lit/tests/googletest-upstream-format.py
llvm/utils/lit/tests/ignore-fail.py
llvm/utils/lit/tests/lit-opts.py
llvm/utils/lit/tests/lit.cfg
llvm/utils/lit/tests/max-failures.py
llvm/utils/lit/tests/progress-bar.py
llvm/utils/lit/tests/reorder.py
llvm/utils/lit/tests/shtest-env.py
llvm/utils/lit/tests/shtest-format-argv0.py
llvm/utils/lit/tests/shtest-format.py
llvm/utils/lit/tests/shtest-inject.py
llvm/utils/lit/tests/shtest-keyword-parse-errors.py
llvm/utils/lit/tests/shtest-not.py
llvm/utils/lit/tests/shtest-output-printing.py
llvm/utils/lit/tests/shtest-recursive-substitution.py
llvm/utils/lit/tests/shtest-run-at-line.py
llvm/utils/lit/tests/shtest-shell.py
llvm/utils/lit/tests/test-data-micro.py
llvm/utils/lit/tests/test-data.py
llvm/utils/lit/tests/test-output-micro.py
llvm/utils/lit/tests/test-output.py
Removed:
################################################################################
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 9c48d3abb8df7..29051389a8fdf 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -168,7 +168,8 @@ SELECTION OPTIONS
-----------------
By default, `lit` will run failing tests first, then run tests in descending
-execution time order to optimize concurrency.
+execution time order to optimize concurrency. The execution order can be
+changed using the :option:`--order` option.
The timing data is stored in the `test_exec_root` in a file named
`.lit_test_times.txt`. If this file does not exist, then `lit` checks the
@@ -176,7 +177,8 @@ The timing data is stored in the `test_exec_root` in a file named
.. option:: --shuffle
- Run the tests in a random order, not failing/slowest first.
+ Run the tests in a random order, not failing/slowest first. Deprecated,
+ use :option:`--order` instead.
.. option:: --max-failures N
@@ -204,6 +206,19 @@ The timing data is stored in the `test_exec_root` in a file named
testsuites, for parallel execution on separate machines (say in a large
testing farm).
+.. option:: --order={lexical,random,smart}
+
+ Define the order in which tests are run. The supported values are:
+
+ - lexical - tests will be run in lexical order according to the test file
+ path. This option is useful when predictable test order is desired.
+
+ - random - tests will be run in random order.
+
+ - smart - tests that failed previously will be run first, then the remaining
+ tests, all in descending execution time order. This is the default as it
+ optimizes concurrency.
+
.. option:: --run-shard=N
Select which shard to run, assuming the ``--num-shards=M`` option was
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index aa83c7ddad4e6..41e63622b0657 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -8,9 +8,11 @@
import lit.util
+ at enum.unique
class TestOrder(enum.Enum):
- DEFAULT = enum.auto()
- RANDOM = enum.auto()
+ LEXICAL = 'lexical'
+ RANDOM = 'random'
+ SMART = 'smart'
def parse_args():
@@ -153,11 +155,17 @@ def parse_args():
metavar="N",
help="Maximum time to spend testing (in seconds)",
type=_positive_int)
+ selection_group.add_argument("--order",
+ choices=[x.value for x in TestOrder],
+ default=TestOrder.SMART,
+ help="Test order to use (default: smart)")
selection_group.add_argument("--shuffle",
- help="Run tests in random order",
- action="store_true")
+ dest="order",
+ help="Run tests in random order (DEPRECATED: use --order=random)",
+ action="store_const",
+ const=TestOrder.RANDOM)
selection_group.add_argument("-i", "--incremental",
- help="Run failed tests first (DEPRECATED: now always enabled)",
+ help="Run failed tests first (DEPRECATED: use --order=smart)",
action="store_true")
selection_group.add_argument("--filter",
metavar="REGEX",
@@ -218,11 +226,6 @@ def parse_args():
if opts.incremental:
print('WARNING: --incremental is deprecated. Failing tests now always run first.')
- if opts.shuffle:
- opts.order = TestOrder.RANDOM
- else:
- opts.order = TestOrder.DEFAULT
-
if opts.numShards or opts.runShard:
if not opts.numShards or not opts.runShard:
parser.error("--num-shards and --run-shard must be used together")
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 6924aee3d5148..73747360601a5 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -166,11 +166,14 @@ def print_discovered(tests, show_suites, show_tests):
def determine_order(tests, order):
from lit.cl_arguments import TestOrder
- if order == TestOrder.RANDOM:
+ enum_order = TestOrder(order)
+ if enum_order == TestOrder.RANDOM:
import random
random.shuffle(tests)
+ elif enum_order == TestOrder.LEXICAL:
+ tests.sort(key=lambda t: t.getFullName())
else:
- assert order == TestOrder.DEFAULT, 'Unknown TestOrder value'
+ assert enum_order == TestOrder.SMART, 'Unknown TestOrder value'
tests.sort(key=lambda t: (not t.previous_failure, -t.previous_elapsed, t.getFullName()))
diff --git a/llvm/utils/lit/tests/allow-retries.py b/llvm/utils/lit/tests/allow-retries.py
index 4877b86178566..b8abe0ba4fee1 100644
--- a/llvm/utils/lit/tests/allow-retries.py
+++ b/llvm/utils/lit/tests/allow-retries.py
@@ -4,32 +4,32 @@
# only succeed the fourth time it is retried.
#
# RUN: rm -f %t.counter
-# RUN: %{lit} -j 1 %{inputs}/allow-retries/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s
+# RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s
# CHECK-TEST1: Passed With Retry: 1
# Test that a per-file ALLOW_RETRIES overwrites the config-wide test_retry_attempts property, if any.
#
# RUN: rm -f %t.counter
-# RUN: %{lit} -j 1 %{inputs}/allow-retries/succeeds-within-limit.py -Dtest_retry_attempts=2 -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST2 %s
+# RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dtest_retry_attempts=2 -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST2 %s
# CHECK-TEST2: Passed With Retry: 1
# This test does not succeed within the allowed retry limit
#
-# RUN: not %{lit} -j 1 %{inputs}/allow-retries/does-not-succeed-within-limit.py | FileCheck --check-prefix=CHECK-TEST3 %s
+# RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py | FileCheck --check-prefix=CHECK-TEST3 %s
# CHECK-TEST3: Failed Tests (1):
# CHECK-TEST3: allow-retries :: does-not-succeed-within-limit.py
# This test should be UNRESOLVED since it has more than one ALLOW_RETRIES
# lines, and that is not allowed.
#
-# RUN: not %{lit} -j 1 %{inputs}/allow-retries/more-than-one-allow-retries-lines.py | FileCheck --check-prefix=CHECK-TEST4 %s
+# RUN: not %{lit} %{inputs}/allow-retries/more-than-one-allow-retries-lines.py | FileCheck --check-prefix=CHECK-TEST4 %s
# CHECK-TEST4: Unresolved Tests (1):
# CHECK-TEST4: allow-retries :: more-than-one-allow-retries-lines.py
# This test does not provide a valid integer to the ALLOW_RETRIES keyword.
# It should be unresolved.
#
-# RUN: not %{lit} -j 1 %{inputs}/allow-retries/not-a-valid-integer.py | FileCheck --check-prefix=CHECK-TEST5 %s
+# RUN: not %{lit} %{inputs}/allow-retries/not-a-valid-integer.py | FileCheck --check-prefix=CHECK-TEST5 %s
# CHECK-TEST5: Unresolved Tests (1):
# CHECK-TEST5: allow-retries :: not-a-valid-integer.py
@@ -37,5 +37,5 @@
# when no ALLOW_RETRIES keyword is present.
#
# RUN: rm -f %t.counter
-# RUN: %{lit} -j 1 %{inputs}/test_retry_attempts/test.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST6 %s
+# RUN: %{lit} %{inputs}/test_retry_attempts/test.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST6 %s
# CHECK-TEST6: Passed With Retry: 1
diff --git a/llvm/utils/lit/tests/custom-result-category.py b/llvm/utils/lit/tests/custom-result-category.py
index d4608b39a1c7d..f147f6b52dafa 100644
--- a/llvm/utils/lit/tests/custom-result-category.py
+++ b/llvm/utils/lit/tests/custom-result-category.py
@@ -1,10 +1,7 @@
# UNSUPPORTED: system-windows
# Test lit.main.add_result_category() extension API.
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/custom-result-category/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 %{inputs}/custom-result-category | FileCheck %s
+# RUN: not %{lit} %{inputs}/custom-result-category | FileCheck %s
# CHECK: CUSTOM_PASS: custom-result-category :: test1.txt
# CHECK: CUSTOM_FAILURE: custom-result-category :: test2.txt
diff --git a/llvm/utils/lit/tests/discovery.py b/llvm/utils/lit/tests/discovery.py
index 39743bfd7c58d..221a634ce4354 100644
--- a/llvm/utils/lit/tests/discovery.py
+++ b/llvm/utils/lit/tests/discovery.py
@@ -1,7 +1,7 @@
# Check the basic discovery process, including a sub-suite.
#
# RUN: %{lit} %{inputs}/discovery \
-# RUN: -j 1 --debug --show-tests --show-suites \
+# RUN: --debug --show-tests --show-suites \
# RUN: -v > %t.out 2> %t.err
# RUN: FileCheck --check-prefix=CHECK-BASIC-OUT < %t.out %s
# RUN: FileCheck --check-prefix=CHECK-BASIC-ERR < %t.err %s
@@ -56,7 +56,7 @@
# RUN: %{lit} \
# RUN: %{inputs}/discovery/subdir/test-three.py \
# RUN: %{inputs}/discovery/subsuite/test-one.txt \
-# RUN: -j 1 --show-tests --show-suites -v > %t.out
+# RUN: --show-tests --show-suites -v > %t.out
# RUN: FileCheck --check-prefix=CHECK-DIRECT-TEST < %t.out %s
#
# CHECK-DIRECT-TEST: -- Available Tests --
@@ -65,7 +65,7 @@
# Check discovery when config files end in .py
# RUN: %{lit} %{inputs}/py-config-discovery \
-# RUN: -j 1 --debug --show-tests --show-suites \
+# RUN: --debug --show-tests --show-suites \
# RUN: -v > %t.out 2> %t.err
# RUN: FileCheck --check-prefix=CHECK-PYCONFIG-OUT < %t.out %s
# RUN: FileCheck --check-prefix=CHECK-PYCONFIG-ERR < %t.err %s
@@ -95,7 +95,7 @@
# Check discovery when using an exec path.
#
# RUN: %{lit} %{inputs}/exec-discovery \
-# RUN: -j 1 --debug --show-tests --show-suites \
+# RUN: --debug --show-tests --show-suites \
# RUN: -v > %t.out 2> %t.err
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-OUT < %t.out %s
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-ERR < %t.err %s
@@ -128,7 +128,7 @@
#
# RUN: %{lit} \
# RUN: %{inputs}/exec-discovery/subdir/test-three.py \
-# RUN: -j 1 --show-tests --show-suites -v > %t.out
+# RUN: --show-tests --show-suites -v > %t.out
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-DIRECT-TEST < %t.out %s
#
# CHECK-ASEXEC-DIRECT-TEST: -- Available Tests --
@@ -138,7 +138,7 @@
# indirectly (e.g. when the directory containing the test is specified).
#
# RUN: not %{lit} \
-# RUN: %{inputs}/discovery/test.not-txt -j 1 2>%t.err
+# RUN: %{inputs}/discovery/test.not-txt 2>%t.err
# RUN: FileCheck --check-prefix=CHECK-ERROR-INDIRECT-RUN-CHECK < %t.err %s
#
# CHECK-ERROR-INDIRECT-RUN-CHECK: error: 'top-level-suite :: test.not-txt' would not be run indirectly
@@ -146,11 +146,11 @@
# Check that no error is emitted with --no-indirectly-run-check.
#
# RUN: %{lit} \
-# RUN: %{inputs}/discovery/test.not-txt -j 1 --no-indirectly-run-check
+# RUN: %{inputs}/discovery/test.not-txt --no-indirectly-run-check
# Check that a standalone test with no suffixes set is run without any errors.
#
-# RUN: %{lit} %{inputs}/standalone-tests/true.txt -j 1 > %t.out
+# RUN: %{lit} %{inputs}/standalone-tests/true.txt > %t.out
# RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s
#
# CHECK-STANDALONE: PASS: Standalone tests :: true.txt
@@ -158,7 +158,7 @@
# Check that an error is produced if suffixes variable is set for a suite with
# standalone tests.
#
-# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes -j 1 2> %t.err
+# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes 2> %t.err
# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s
#
# CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes
@@ -166,14 +166,14 @@
# Check that an error is produced if excludes variable is set for a suite with
# standalone tests.
#
-# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes -j 1 2> %t.err
+# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes 2> %t.err
# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s
#
# CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes
# Check that no discovery is done for testsuite with standalone tests.
#
-# RUN: not %{lit} %{inputs}/standalone-tests -j 1 2>%t.err
+# RUN: not %{lit} %{inputs}/standalone-tests 2>%t.err
# RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s
#
# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)
@@ -183,14 +183,14 @@
#
# RUN: %{lit} \
# RUN: %{inputs}/exec-discovery-in-tree/obj/ \
-# RUN: -j 1 --show-tests --show-suites -v > %t.out
+# RUN: --show-tests --show-suites -v > %t.out
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s
#
# Try it again after cd'ing into the test suite using a short relative path.
#
# RUN: cd %{inputs}/exec-discovery-in-tree/obj/
# RUN: %{lit} . \
-# RUN: -j 1 --show-tests --show-suites -v > %t.out
+# RUN: --show-tests --show-suites -v > %t.out
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s
#
# CHECK-ASEXEC-INTREE: exec-discovery-in-tree-suite - 1 tests
diff --git a/llvm/utils/lit/tests/googletest-discovery-failed.py b/llvm/utils/lit/tests/googletest-discovery-failed.py
index 0e11e78078f70..bc84c8f03226a 100644
--- a/llvm/utils/lit/tests/googletest-discovery-failed.py
+++ b/llvm/utils/lit/tests/googletest-discovery-failed.py
@@ -1,6 +1,6 @@
# Check for correct error message when discovery of tests fails.
#
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-discovery-failed > %t.cmd.out
+# RUN: not %{lit} -v %{inputs}/googletest-discovery-failed > %t.cmd.out
# RUN: FileCheck < %t.cmd.out %s
diff --git a/llvm/utils/lit/tests/googletest-format.py b/llvm/utils/lit/tests/googletest-format.py
index c5216f5427c36..b960d0cdc64e8 100644
--- a/llvm/utils/lit/tests/googletest-format.py
+++ b/llvm/utils/lit/tests/googletest-format.py
@@ -1,9 +1,6 @@
# Check the various features of the GoogleTest format.
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/googletest-format/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-format > %t.out
+# RUN: not %{lit} -v %{inputs}/googletest-format > %t.out
# FIXME: Temporarily dump test output so we can debug failing tests on
# buildbots.
# RUN: cat %t.out
diff --git a/llvm/utils/lit/tests/googletest-timeout.py b/llvm/utils/lit/tests/googletest-timeout.py
index adfc4d9ae89af..b7b9b64d4ed56 100644
--- a/llvm/utils/lit/tests/googletest-timeout.py
+++ b/llvm/utils/lit/tests/googletest-timeout.py
@@ -6,14 +6,14 @@
# Check that the per test timeout is enforced when running GTest tests.
#
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
+# RUN: not %{lit} -v %{inputs}/googletest-timeout \
# RUN: --filter=InfiniteLoopSubTest --timeout=1 > %t.cmd.out
# RUN: FileCheck --check-prefix=CHECK-INF < %t.cmd.out %s
# Check that the per test timeout is enforced when running GTest tests via
# the configuration file
#
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
+# RUN: not %{lit} -v %{inputs}/googletest-timeout \
# RUN: --filter=InfiniteLoopSubTest --param set_timeout=1 \
# RUN: > %t.cfgset.out
# RUN: FileCheck --check-prefix=CHECK-INF < %t.cfgset.out %s
@@ -34,7 +34,7 @@
# 3600 second timeout.
###############################################################################
-# RUN: %{lit} -j 1 -v %{inputs}/googletest-timeout \
+# RUN: %{lit} -v %{inputs}/googletest-timeout \
# RUN: --filter=QuickSubTest --timeout=3600 > %t.cmd.out
# RUN: FileCheck --check-prefix=CHECK-QUICK < %t.cmd.out %s
@@ -43,7 +43,7 @@
# Test per test timeout via a config file and on the command line.
# The value set on the command line should override the config file.
-# RUN: %{lit} -j 1 -v %{inputs}/googletest-timeout --filter=QuickSubTest \
+# RUN: %{lit} -v %{inputs}/googletest-timeout --filter=QuickSubTest \
# RUN: --param set_timeout=1 --timeout=3600 \
# RUN: > %t.cmdover.out 2> %t.cmdover.err
# RUN: FileCheck --check-prefix=CHECK-QUICK < %t.cmdover.out %s
diff --git a/llvm/utils/lit/tests/googletest-upstream-format.py b/llvm/utils/lit/tests/googletest-upstream-format.py
index 8811b1eb30861..cf0f73e2d31ed 100644
--- a/llvm/utils/lit/tests/googletest-upstream-format.py
+++ b/llvm/utils/lit/tests/googletest-upstream-format.py
@@ -1,9 +1,6 @@
# Check the various features of the GoogleTest format.
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/googletest-upstream-format/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-upstream-format > %t.out
+# RUN: not %{lit} -v %{inputs}/googletest-upstream-format > %t.out
# RUN: FileCheck < %t.out %s
#
# END.
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 63c34516226d1..0644959828a3c 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -1,8 +1,8 @@
# Check that --ignore-fail produces exit status 0 despite various kinds of
# test failures but doesn't otherwise suppress those failures.
-# RUN: not %{lit} -j 1 %{inputs}/ignore-fail | FileCheck %s
-# RUN: %{lit} -j 1 --ignore-fail %{inputs}/ignore-fail | FileCheck %s
+# RUN: not %{lit} %{inputs}/ignore-fail | FileCheck %s
+# RUN: %{lit} --ignore-fail %{inputs}/ignore-fail | FileCheck %s
# END.
diff --git a/llvm/utils/lit/tests/lit-opts.py b/llvm/utils/lit/tests/lit-opts.py
index 83600388558b7..d292ca74f3b9b 100644
--- a/llvm/utils/lit/tests/lit-opts.py
+++ b/llvm/utils/lit/tests/lit-opts.py
@@ -1,25 +1,25 @@
# Check cases where LIT_OPTS has no effect.
#
-# RUN: %{lit} -j 1 -s %{inputs}/lit-opts | FileCheck %s
-# RUN: env LIT_OPTS= %{lit} -j 1 -s %{inputs}/lit-opts | FileCheck %s
-# RUN: env LIT_OPTS=-s %{lit} -j 1 -s %{inputs}/lit-opts | FileCheck %s
+# RUN: %{lit} -s %{inputs}/lit-opts | FileCheck %s
+# RUN: env LIT_OPTS= %{lit} -s %{inputs}/lit-opts | FileCheck %s
+# RUN: env LIT_OPTS=-s %{lit} -s %{inputs}/lit-opts | FileCheck %s
# Check that LIT_OPTS can override command-line options.
#
# RUN: env LIT_OPTS=-a \
-# RUN: %{lit} -j 1 -s %{inputs}/lit-opts \
+# RUN: %{lit} -s %{inputs}/lit-opts \
# RUN: | FileCheck -check-prefix=SHOW-ALL -DVAR= %s
# Check that LIT_OPTS understands multiple options with arbitrary spacing.
#
# RUN: env LIT_OPTS='-a -v -Dvar=foobar' \
-# RUN: %{lit} -j 1 -s %{inputs}/lit-opts \
+# RUN: %{lit} -s %{inputs}/lit-opts \
# RUN: | FileCheck -check-prefix=SHOW-ALL -DVAR=foobar %s
# Check that LIT_OPTS parses shell-like quotes and escapes.
#
# RUN: env LIT_OPTS='-a -v -Dvar="foo bar"\ baz' \
-# RUN: %{lit} -j 1 -s %{inputs}/lit-opts \
+# RUN: %{lit} -s %{inputs}/lit-opts \
# RUN: | FileCheck -check-prefix=SHOW-ALL -DVAR="foo bar baz" %s
# CHECK: Testing: 1 tests
diff --git a/llvm/utils/lit/tests/lit.cfg b/llvm/utils/lit/tests/lit.cfg
index c5c6d501c7b4b..fa11e8f9643a6 100644
--- a/llvm/utils/lit/tests/lit.cfg
+++ b/llvm/utils/lit/tests/lit.cfg
@@ -57,11 +57,14 @@ for attribute in ('llvm_tools_dir', 'lit_tools_dir'):
# suites in %{inputs}. This test suite's results are then determined in part
# by %{lit}'s textual output, which includes the output of FileCheck calls
# within %{inputs}'s test suites. Thus, %{lit} clears environment variables
-# that can affect FileCheck's output.
+# that can affect FileCheck's output. It also includes "--order=lexical -j1"
+# to ensure predictable test order, as it is often required for FileCheck
+# matches.
config.substitutions.append(('%{inputs}', os.path.join(
config.test_source_root, 'Inputs')))
-config.substitutions.append(('%{lit}',
- "{env} %{{python}} {lit}".format(
+config.substitutions.append(('%{lit}', '%{lit-no-order-opt} --order=lexical'))
+config.substitutions.append(('%{lit-no-order-opt}',
+ "{env} %{{python}} {lit} -j1".format(
env="env -u FILECHECK_OPTS",
lit=os.path.join(lit_path, 'lit.py'))))
config.substitutions.append(('%{python}', '"%s"' % (sys.executable)))
diff --git a/llvm/utils/lit/tests/max-failures.py b/llvm/utils/lit/tests/max-failures.py
index 0361fe53bf5e1..4ddf043337967 100644
--- a/llvm/utils/lit/tests/max-failures.py
+++ b/llvm/utils/lit/tests/max-failures.py
@@ -2,10 +2,10 @@
# Check the behavior of --max-failures option.
#
-# RUN: not %{lit} -j 1 %{inputs}/max-failures > %t.out 2>&1
-# RUN: not %{lit} --max-failures=1 -j 1 %{inputs}/max-failures >> %t.out 2>&1
-# RUN: not %{lit} --max-failures=2 -j 1 %{inputs}/max-failures >> %t.out 2>&1
-# RUN: not %{lit} --max-failures=0 -j 1 %{inputs}/max-failures 2>> %t.out
+# RUN: not %{lit} %{inputs}/max-failures > %t.out 2>&1
+# RUN: not %{lit} --max-failures=1 %{inputs}/max-failures >> %t.out 2>&1
+# RUN: not %{lit} --max-failures=2 %{inputs}/max-failures >> %t.out 2>&1
+# RUN: not %{lit} --max-failures=0 %{inputs}/max-failures 2>> %t.out
# RUN: FileCheck < %t.out %s
#
diff --git a/llvm/utils/lit/tests/progress-bar.py b/llvm/utils/lit/tests/progress-bar.py
index e6dafffae8790..89f4cba60d6ae 100644
--- a/llvm/utils/lit/tests/progress-bar.py
+++ b/llvm/utils/lit/tests/progress-bar.py
@@ -1,9 +1,6 @@
# Check the simple progress bar.
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/progress-bar/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -s %{inputs}/progress-bar > %t.out
+# RUN: not %{lit} -s %{inputs}/progress-bar > %t.out
# RUN: FileCheck < %t.out %s
#
# CHECK: Testing:
diff --git a/llvm/utils/lit/tests/reorder.py b/llvm/utils/lit/tests/reorder.py
index cac0a4eecaa67..f911de21ed316 100644
--- a/llvm/utils/lit/tests/reorder.py
+++ b/llvm/utils/lit/tests/reorder.py
@@ -1,7 +1,7 @@
## Check that we can reorder test runs.
# RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig
-# RUN: %{lit} -j1 %{inputs}/reorder > %t.out
+# RUN: %{lit-no-order-opt} %{inputs}/reorder > %t.out
# RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.new
# RUN: cp %{inputs}/reorder/.lit_test_times.txt.orig %{inputs}/reorder/.lit_test_times.txt
# RUN: not
diff %{inputs}/reorder/.lit_test_times.txt.new %{inputs}/reorder/.lit_test_times.txt.orig
diff --git a/llvm/utils/lit/tests/shtest-env.py b/llvm/utils/lit/tests/shtest-env.py
index ead217ace3616..f2e8216f7f4a7 100644
--- a/llvm/utils/lit/tests/shtest-env.py
+++ b/llvm/utils/lit/tests/shtest-env.py
@@ -1,9 +1,6 @@
# Check the env command
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/shtest-env/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-env \
+# RUN: not %{lit} -a -v %{inputs}/shtest-env \
# RUN: | FileCheck -match-full-lines %s
#
# END.
diff --git a/llvm/utils/lit/tests/shtest-format-argv0.py b/llvm/utils/lit/tests/shtest-format-argv0.py
index 28f9acaa3322d..754410a3f38b9 100644
--- a/llvm/utils/lit/tests/shtest-format-argv0.py
+++ b/llvm/utils/lit/tests/shtest-format-argv0.py
@@ -7,7 +7,7 @@
# and is not installed under PATH by default.
# UNSUPPORTED: system-aix
#
-# RUN: %{lit} -j 1 -v %{inputs}/shtest-format-argv0 | FileCheck %s
+# RUN: %{lit} -v %{inputs}/shtest-format-argv0 | FileCheck %s
# CHECK: -- Testing:
# CHECK: PASS: shtest-format-argv0 :: argv0.txt
diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 4d8004db267a6..f39d779238492 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -1,10 +1,7 @@
# Check the various features of the ShTest format.
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/shtest-format/.lit_test_times.txt
-
# RUN: rm -f %t.xml
-# RUN: not %{lit} -j 1 -v %{inputs}/shtest-format --xunit-xml-output %t.xml > %t.out
+# RUN: not %{lit} -v %{inputs}/shtest-format --xunit-xml-output %t.xml > %t.out
# RUN: FileCheck < %t.out %s
# RUN: FileCheck --check-prefix=XUNIT < %t.xml %s
diff --git a/llvm/utils/lit/tests/shtest-inject.py b/llvm/utils/lit/tests/shtest-inject.py
index f6df447ddde42..c6fa799ac3bda 100644
--- a/llvm/utils/lit/tests/shtest-inject.py
+++ b/llvm/utils/lit/tests/shtest-inject.py
@@ -1,6 +1,6 @@
# Check that we can inject commands at the beginning of a ShTest.
-# RUN: %{lit} -j 1 %{inputs}/shtest-inject/test-empty.txt --show-all | FileCheck --check-prefix=CHECK-TEST1 %s
+# RUN: %{lit} %{inputs}/shtest-inject/test-empty.txt --show-all | FileCheck --check-prefix=CHECK-TEST1 %s
#
# CHECK-TEST1: Script:
# CHECK-TEST1: --
@@ -13,7 +13,7 @@
#
# CHECK-TEST1: Passed: 1
-# RUN: %{lit} -j 1 %{inputs}/shtest-inject/test-one.txt --show-all | FileCheck --check-prefix=CHECK-TEST2 %s
+# RUN: %{lit} %{inputs}/shtest-inject/test-one.txt --show-all | FileCheck --check-prefix=CHECK-TEST2 %s
#
# CHECK-TEST2: Script:
# CHECK-TEST2: --
@@ -28,7 +28,7 @@
#
# CHECK-TEST2: Passed: 1
-# RUN: %{lit} -j 1 %{inputs}/shtest-inject/test-many.txt --show-all | FileCheck --check-prefix=CHECK-TEST3 %s
+# RUN: %{lit} %{inputs}/shtest-inject/test-many.txt --show-all | FileCheck --check-prefix=CHECK-TEST3 %s
#
# CHECK-TEST3: Script:
# CHECK-TEST3: --
diff --git a/llvm/utils/lit/tests/shtest-keyword-parse-errors.py b/llvm/utils/lit/tests/shtest-keyword-parse-errors.py
index 78cbf81a3e6a0..0d4c693f9bd65 100644
--- a/llvm/utils/lit/tests/shtest-keyword-parse-errors.py
+++ b/llvm/utils/lit/tests/shtest-keyword-parse-errors.py
@@ -1,7 +1,4 @@
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/shtest-keyword-parse-errors/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-keyword-parse-errors > %t.out
+# RUN: not %{lit} -vv %{inputs}/shtest-keyword-parse-errors > %t.out
# RUN: FileCheck -input-file %t.out %s
#
# END.
diff --git a/llvm/utils/lit/tests/shtest-not.py b/llvm/utils/lit/tests/shtest-not.py
index 2b8a69a0f66f0..53bd6356ad930 100644
--- a/llvm/utils/lit/tests/shtest-not.py
+++ b/llvm/utils/lit/tests/shtest-not.py
@@ -1,9 +1,6 @@
# Check the not command
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/shtest-not/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-not \
+# RUN: not %{lit} -a -v %{inputs}/shtest-not \
# RUN: | FileCheck -match-full-lines %s
#
# END.
diff --git a/llvm/utils/lit/tests/shtest-output-printing.py b/llvm/utils/lit/tests/shtest-output-printing.py
index 80304258fd954..d5ec413fa04be 100644
--- a/llvm/utils/lit/tests/shtest-output-printing.py
+++ b/llvm/utils/lit/tests/shtest-output-printing.py
@@ -1,6 +1,6 @@
# Check the various features of the ShTest format.
#
-# RUN: not %{lit} -j 1 -v %{inputs}/shtest-output-printing > %t.out
+# RUN: not %{lit} -v %{inputs}/shtest-output-printing > %t.out
# RUN: FileCheck --input-file %t.out %s
#
# END.
diff --git a/llvm/utils/lit/tests/shtest-recursive-substitution.py b/llvm/utils/lit/tests/shtest-recursive-substitution.py
index d446c422139bc..48f4b5b124911 100644
--- a/llvm/utils/lit/tests/shtest-recursive-substitution.py
+++ b/llvm/utils/lit/tests/shtest-recursive-substitution.py
@@ -1,27 +1,27 @@
# Check that the config.recursiveExpansionLimit is picked up and will cause
# lit substitutions to be expanded recursively.
-# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/substitutes-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST1 %s
+# RUN: %{lit} %{inputs}/shtest-recursive-substitution/substitutes-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST1 %s
# CHECK-TEST1: PASS: substitutes-within-limit :: test.py
# CHECK-TEST1: $ "echo" "STOP"
-# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/does-not-substitute-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST2 %s
+# RUN: not %{lit} %{inputs}/shtest-recursive-substitution/does-not-substitute-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST2 %s
# CHECK-TEST2: UNRESOLVED: does-not-substitute-within-limit :: test.py
# CHECK-TEST2: ValueError: Recursive substitution of
-# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/does-not-substitute-no-limit --show-all | FileCheck --check-prefix=CHECK-TEST3 %s
+# RUN: %{lit} %{inputs}/shtest-recursive-substitution/does-not-substitute-no-limit --show-all | FileCheck --check-prefix=CHECK-TEST3 %s
# CHECK-TEST3: PASS: does-not-substitute-no-limit :: test.py
# CHECK-TEST3: $ "echo" "%rec4"
-# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/not-an-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST4 %s
+# RUN: not %{lit} %{inputs}/shtest-recursive-substitution/not-an-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST4 %s
# CHECK-TEST4: recursiveExpansionLimit must be either None or an integer
-# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/negative-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST5 %s
+# RUN: not %{lit} %{inputs}/shtest-recursive-substitution/negative-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST5 %s
# CHECK-TEST5: recursiveExpansionLimit must be a non-negative integer
-# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/set-to-none --show-all | FileCheck --check-prefix=CHECK-TEST6 %s
+# RUN: %{lit} %{inputs}/shtest-recursive-substitution/set-to-none --show-all | FileCheck --check-prefix=CHECK-TEST6 %s
# CHECK-TEST6: PASS: set-to-none :: test.py
-# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/escaping --show-all | FileCheck --check-prefix=CHECK-TEST7 %s
+# RUN: %{lit} %{inputs}/shtest-recursive-substitution/escaping --show-all | FileCheck --check-prefix=CHECK-TEST7 %s
# CHECK-TEST7: PASS: escaping :: test.py
# CHECK-TEST7: $ "echo" "%s" "%s" "%%s"
diff --git a/llvm/utils/lit/tests/shtest-run-at-line.py b/llvm/utils/lit/tests/shtest-run-at-line.py
index bf47e6499cd2f..ccd85b505b7e4 100644
--- a/llvm/utils/lit/tests/shtest-run-at-line.py
+++ b/llvm/utils/lit/tests/shtest-run-at-line.py
@@ -1,10 +1,7 @@
# Check that -vv makes the line number of the failing RUN command clear.
# (-v is actually sufficient in the case of the internal shell.)
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/shtest-run-at-line/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
+# RUN: not %{lit} -vv %{inputs}/shtest-run-at-line > %t.out
# RUN: FileCheck --input-file %t.out %s
#
# END.
diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py
index 13cf05f304fad..595960ce79e76 100644
--- a/llvm/utils/lit/tests/shtest-shell.py
+++ b/llvm/utils/lit/tests/shtest-shell.py
@@ -1,9 +1,6 @@
# Check the internal shell handling component of the ShTest format.
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/shtest-shell/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out
+# RUN: not %{lit} -v %{inputs}/shtest-shell > %t.out
# FIXME: Temporarily dump test output so we can debug failing tests on
# buildbots.
# RUN: cat %t.out
@@ -11,9 +8,7 @@
#
# Test again in non-UTF shell to catch potential errors with python 2 seen
# on stdout-encoding.txt
-# FIXME: lit's testing sets source_root == exec_root which complicates running lit more than once per test.
-# RUN: rm -f %{inputs}/shtest-shell/.lit_test_times.txt
-# RUN: env PYTHONIOENCODING=ascii not %{lit} -j 1 -a %{inputs}/shtest-shell > %t.ascii.out
+# RUN: env PYTHONIOENCODING=ascii not %{lit} -a %{inputs}/shtest-shell > %t.ascii.out
# FIXME: Temporarily dump test output so we can debug failing tests on
# buildbots.
# RUN: cat %t.ascii.out
diff --git a/llvm/utils/lit/tests/test-data-micro.py b/llvm/utils/lit/tests/test-data-micro.py
index 634139e233f97..7aea434489d67 100644
--- a/llvm/utils/lit/tests/test-data-micro.py
+++ b/llvm/utils/lit/tests/test-data-micro.py
@@ -1,7 +1,7 @@
# Test features related to formats which support reporting additional test data.
# and multiple test results.
-# RUN: %{lit} -j 1 -v %{inputs}/test-data-micro | FileCheck %s
+# RUN: %{lit} -v %{inputs}/test-data-micro | FileCheck %s
# CHECK: -- Testing:
diff --git a/llvm/utils/lit/tests/test-data.py b/llvm/utils/lit/tests/test-data.py
index 54909d7338e96..628a319dd4f52 100644
--- a/llvm/utils/lit/tests/test-data.py
+++ b/llvm/utils/lit/tests/test-data.py
@@ -1,6 +1,6 @@
# Test features related to formats which support reporting additional test data.
-# RUN: %{lit} -j 1 -v %{inputs}/test-data > %t.out
+# RUN: %{lit} -v %{inputs}/test-data > %t.out
# RUN: FileCheck < %t.out %s
# CHECK: -- Testing:
diff --git a/llvm/utils/lit/tests/test-output-micro.py b/llvm/utils/lit/tests/test-output-micro.py
index 4357fe88f905f..2f49cfbfb126c 100644
--- a/llvm/utils/lit/tests/test-output-micro.py
+++ b/llvm/utils/lit/tests/test-output-micro.py
@@ -1,4 +1,4 @@
-# RUN: %{lit} -j 1 -v %{inputs}/test-data-micro --output %t.results.out
+# RUN: %{lit} -v %{inputs}/test-data-micro --output %t.results.out
# RUN: FileCheck < %t.results.out %s
# RUN: rm %t.results.out
diff --git a/llvm/utils/lit/tests/test-output.py b/llvm/utils/lit/tests/test-output.py
index a50442741b9d5..b6640be96547d 100644
--- a/llvm/utils/lit/tests/test-output.py
+++ b/llvm/utils/lit/tests/test-output.py
@@ -1,4 +1,4 @@
-# RUN: %{lit} -j 1 -v %{inputs}/test-data --output %t.results.out > %t.out
+# RUN: %{lit} -v %{inputs}/test-data --output %t.results.out > %t.out
# RUN: FileCheck < %t.results.out %s
# CHECK: {
More information about the llvm-commits
mailing list