[llvm] r373794 - [lit] Use better name for "test in parallel" concept
Julian Lettner via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 4 14:40:20 PDT 2019
Author: yln
Date: Fri Oct 4 14:40:20 2019
New Revision: 373794
URL: http://llvm.org/viewvc/llvm-project?rev=373794&view=rev
Log:
[lit] Use better name for "test in parallel" concept
In the past, lit used threads to run tests in parallel. Today we use
`multiprocessing.Pool`, which uses processes. Let's stay more abstract
and use "worker" everywhere.
Reviewed By: rnk
Differential Revision: https://reviews.llvm.org/D68475
Modified:
llvm/trunk/utils/lit/lit/main.py
llvm/trunk/utils/lit/lit/run.py
llvm/trunk/utils/lit/tests/discovery.py
llvm/trunk/utils/lit/tests/parallelism-groups.py
Modified: llvm/trunk/utils/lit/lit/main.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/main.py?rev=373794&r1=373793&r2=373794&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/main.py (original)
+++ llvm/trunk/utils/lit/lit/main.py Fri Oct 4 14:40:20 2019
@@ -209,8 +209,8 @@ def main_with_tmp(builtinParameters):
parser.add_argument("--version", dest="show_version",
help="Show version and exit",
action="store_true", default=False)
- parser.add_argument("-j", "--threads", dest="numThreads", metavar="N",
- help="Number of testing threads",
+ parser.add_argument("-j", "--threads", "--workers", dest="numWorkers", metavar="N",
+ help="Number of workers used for testing",
type=int, default=None)
parser.add_argument("--config-prefix", dest="configPrefix",
metavar="NAME", help="Prefix for 'lit' config files",
@@ -334,10 +334,10 @@ def main_with_tmp(builtinParameters):
if not args:
parser.error('No inputs specified')
- if opts.numThreads is None:
- opts.numThreads = lit.util.detectCPUs()
- elif opts.numThreads <= 0:
- parser.error("Option '--threads' or '-j' requires positive integer")
+ if opts.numWorkers is None:
+ opts.numWorkers = lit.util.detectCPUs()
+ elif opts.numWorkers <= 0:
+ parser.error("Option '--workers' or '-j' requires positive integer")
if opts.maxFailures is not None and opts.maxFailures <= 0:
parser.error("Option '--max-failures' requires positive integer")
@@ -480,8 +480,8 @@ def main_with_tmp(builtinParameters):
if opts.maxTests is not None:
run.tests = run.tests[:opts.maxTests]
- # Don't create more threads than tests.
- opts.numThreads = min(len(run.tests), opts.numThreads)
+ # Don't create more workers than tests.
+ opts.numWorkers = min(len(run.tests), opts.numWorkers)
# Because some tests use threads internally, and at least on Linux each
# of these threads counts toward the current process limit, try to
@@ -489,7 +489,7 @@ def main_with_tmp(builtinParameters):
# resource exhaustion.
try:
cpus = lit.util.detectCPUs()
- desired_limit = opts.numThreads * cpus * 2 # the 2 is a safety factor
+ desired_limit = opts.numWorkers * cpus * 2 # the 2 is a safety factor
# Import the resource module here inside this try block because it
# will likely fail on Windows.
@@ -506,8 +506,7 @@ def main_with_tmp(builtinParameters):
pass
extra = (' of %d' % numTotalTests) if (len(run.tests) != numTotalTests) else ''
- threads = 'single process' if (opts.numThreads == 1) else ('%d threads' % opts.numThreads)
- header = '-- Testing: %d%s tests, %s --' % (len(run.tests), extra, threads)
+ header = '-- Testing: %d%s tests, %d workers --' % (len(run.tests), extra, opts.numWorkers)
progressBar = None
if not opts.quiet:
if opts.succinct and opts.useProgressBar:
@@ -523,7 +522,7 @@ def main_with_tmp(builtinParameters):
startTime = time.time()
display = TestingProgressDisplay(opts, len(run.tests), progressBar)
try:
- run.execute_tests(display, opts.numThreads, opts.maxTime)
+ run.execute_tests(display, opts.numWorkers, opts.maxTime)
except KeyboardInterrupt:
sys.exit(2)
display.finish()
Modified: llvm/trunk/utils/lit/lit/run.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/run.py?rev=373794&r1=373793&r2=373794&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/run.py (original)
+++ llvm/trunk/utils/lit/lit/run.py Fri Oct 4 14:40:20 2019
@@ -37,7 +37,7 @@ class Run(object):
multiprocessing.BoundedSemaphore(v)
for k, v in lit_config.parallelism_groups.items()}
- def execute_tests_in_pool(self, jobs, max_time):
+ def _execute_tests_in_pool(self, workers, max_time):
# We need to issue many wait calls, so compute the final deadline and
# subtract time.time() from that as we go along.
deadline = None
@@ -49,7 +49,7 @@ class Run(object):
# interrupts the workers before we make it into our task callback, they
# will each raise a KeyboardInterrupt exception and print to stderr at
# the same time.
- pool = multiprocessing.Pool(jobs, lit.worker.initializer,
+ pool = multiprocessing.Pool(workers, lit.worker.initializer,
(self.lit_config,
self.parallelism_semaphores))
@@ -93,11 +93,11 @@ class Run(object):
finally:
pool.join()
- def execute_tests(self, display, jobs, max_time=None):
+ def execute_tests(self, display, workers, max_time=None):
"""
- execute_tests(display, jobs, [max_time])
+ execute_tests(display, workers, [max_time])
- Execute each of the tests in the run, using up to jobs number of
+ Execute the tests in the run using up to the specified number of
parallel tasks, and inform the display of each individual result. The
provided tests should be a subset of the tests available in this run
object.
@@ -105,10 +105,8 @@ class Run(object):
If max_time is non-None, it should be a time in seconds after which to
stop executing tests.
- The display object will have its update method called with each test as
- it is completed. The calls are guaranteed to be locked with respect to
- one another, but are *not* guaranteed to be called on the same thread as
- this method was invoked on.
+ The display object will have its update method called for each completed
+ test.
Upon completion, each test in the run will have its result
computed. Tests which were not actually executed (for any reason) will
@@ -124,14 +122,14 @@ class Run(object):
self.failure_count = 0
self.hit_max_failures = False
- if jobs == 1:
+ if workers == 1:
for test_index, test in enumerate(self.tests):
lit.worker._execute_test(test, self.lit_config)
self.consume_test_result((test_index, test))
if self.hit_max_failures:
break
else:
- self.execute_tests_in_pool(jobs, max_time)
+ self._execute_tests_in_pool(workers, max_time)
# Mark any tests that weren't run as UNRESOLVED.
for test in self.tests:
Modified: llvm/trunk/utils/lit/tests/discovery.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/tests/discovery.py?rev=373794&r1=373793&r2=373794&view=diff
==============================================================================
--- llvm/trunk/utils/lit/tests/discovery.py (original)
+++ llvm/trunk/utils/lit/tests/discovery.py Fri Oct 4 14:40:20 2019
@@ -29,7 +29,7 @@
# RUN: %{python} %{inputs}/config-map-discovery/driver.py \
# RUN: %{inputs}/config-map-discovery/main-config/lit.cfg \
# RUN: %{inputs}/config-map-discovery/lit.alt.cfg \
-# RUN: --threads=1 --debug --show-tests --show-suites > %t.out 2> %t.err
+# RUN: --workers=1 --debug --show-tests --show-suites > %t.out 2> %t.err
# RUN: FileCheck --check-prefix=CHECK-CONFIG-MAP-OUT < %t.out %s
# RUN: FileCheck --check-prefix=CHECK-CONFIG-MAP-ERR < %t.err %s
Modified: llvm/trunk/utils/lit/tests/parallelism-groups.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/tests/parallelism-groups.py?rev=373794&r1=373793&r2=373794&view=diff
==============================================================================
--- llvm/trunk/utils/lit/tests/parallelism-groups.py (original)
+++ llvm/trunk/utils/lit/tests/parallelism-groups.py Fri Oct 4 14:40:20 2019
@@ -15,7 +15,7 @@
# RUN: %{lit} -j2 %{inputs}/parallelism-groups | FileCheck %s
-# CHECK: -- Testing: 2 tests, 2 threads --
+# CHECK: -- Testing: 2 tests, 2 workers --
# CHECK-DAG: PASS: parallelism-groups :: test1.txt
# CHECK-DAG: PASS: parallelism-groups :: test2.txt
# CHECK: Expected Passes : 2
More information about the llvm-commits
mailing list