[llvm] [lit] Migrate lit to ThreadPoolExecutor (PR #212397)
Prasoon Kumar via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 27 20:48:26 PDT 2026
https://github.com/prasoon054 created https://github.com/llvm/llvm-project/pull/212397
Lit currently runs each test in its own worker process via ProcessPoolExecutor. Move the default execution engine to ThreadPoolExecutor instead: a worker process pays fork/exec plus pickling overhead per test that a single-process, multi-threaded model avoids, and it's groundwork for a possible asyncio backend later. ProcessPoolExecutor stays available behind --use-process-workers.
>From 8df7fec397941d9359aa6606d61b1a101b5c20c8 Mon Sep 17 00:00:00 2001
From: Prasoon Kumar <prasoonkumar054 at gmail.com>
Date: Tue, 28 Jul 2026 06:36:58 +0530
Subject: [PATCH] [lit] Migrate lit to ThreadPoolExecutor
Lit currently runs each test in its own worker process via
ProcessPoolExecutor. Move the default execution engine to
ThreadPoolExecutor instead: a worker process pays fork/exec plus
pickling overhead per test that a single-process, multi-threaded model
avoids, and it's groundwork for a possible asyncio backend later.
ProcessPoolExecutor stays available behind --use-process-workers.
Signed-off-by: Prasoon Kumar <prasoonkumar054 at gmail.com>
---
llvm/utils/lit/lit/LitConfig.py | 2 +
llvm/utils/lit/lit/Test.py | 1 +
llvm/utils/lit/lit/TestRunner.py | 64 +++++++++++++++---
llvm/utils/lit/lit/cl_arguments.py | 7 ++
llvm/utils/lit/lit/formats/googletest.py | 5 +-
llvm/utils/lit/lit/main.py | 1 +
llvm/utils/lit/lit/reports.py | 3 +-
llvm/utils/lit/lit/run.py | 82 +++++++++++++++---------
llvm/utils/lit/lit/util.py | 12 +++-
llvm/utils/lit/lit/worker.py | 7 +-
10 files changed, 138 insertions(+), 46 deletions(-)
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index 1bc466014f93d..53655899eb437 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -41,6 +41,7 @@ def __init__(
parallelism_groups={},
per_test_coverage=False,
gtest_sharding=True,
+ use_process_workers=False,
update_tests=False,
):
# The name of the test runner.
@@ -53,6 +54,7 @@ def __init__(
self.valgrindUserArgs = list(valgrindArgs)
self.noExecute = noExecute
self.debug = debug
+ self.use_process_workers = bool(use_process_workers)
self.isWindows = bool(isWindows)
self.order = order
self.params = dict(params)
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 35da7afc591c0..d58a43a214a21 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -170,6 +170,7 @@ def __init__(
self.elapsed = elapsed
self.start = None
self.pid = None
+ self.tid = None
# The metrics reported by this test.
self.metrics = {}
# The micro-test results reported by this test.
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 97cb54c63be64..550268014a620 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import contextlib
import enum
import os
import pathlib
@@ -70,6 +71,48 @@ def __init__(self, message):
kPdbgRegex = "%dbg\\(([^)'\"]*)\\)((?:.|\\n)*)"
+class _UmaskGate:
+ def __init__(self):
+ self._cv = threading.Condition()
+ self._active_readers = 0
+ self._writer_active = False
+ self._writers_waiting = 0
+
+ @contextlib.contextmanager
+ def read(self):
+ with self._cv:
+ while self._writer_active or self._writers_waiting:
+ self._cv.wait()
+ self._active_readers += 1
+ try:
+ yield
+ finally:
+ with self._cv:
+ self._active_readers -= 1
+ if self._active_readers == 0:
+ self._cv.notify_all()
+
+ @contextlib.contextmanager
+ def write(self):
+ with self._cv:
+ self._writers_waiting += 1
+ try:
+ while self._writer_active or self._active_readers > 0:
+ self._cv.wait()
+ finally:
+ self._writers_waiting -= 1
+ self._writer_active = True
+ try:
+ yield
+ finally:
+ with self._cv:
+ self._writer_active = False
+ self._cv.notify_all()
+
+
+_umask_gate = _UmaskGate()
+
+
def buildPdbgCommand(msg, cmd):
res = f"%dbg({msg}) {cmd}"
assert re.fullmatch(
@@ -463,11 +506,8 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
# os.umask as the umask argument in subprocess.Popen is not
# available before Python 3.9. Once LLVM requires at least Python
# 3.9, this code should be updated to use umask argument.
- old_umask = -1
- if cmd_shenv.umask != -1:
- old_umask = os.umask(cmd_shenv.umask)
- procs.append(
- subprocess.Popen(
+ def spawn():
+ return subprocess.Popen(
args,
cwd=cmd_shenv.cwd,
executable=executable,
@@ -479,9 +519,17 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
universal_newlines=True,
errors="replace",
)
- )
- if old_umask != -1:
- os.umask(old_umask)
+
+ if cmd_shenv.umask != -1:
+ with _umask_gate.write():
+ old_umask = os.umask(cmd_shenv.umask)
+ try:
+ procs.append(spawn())
+ finally:
+ os.umask(old_umask)
+ else:
+ with _umask_gate.read():
+ procs.append(spawn())
proc_not_counts.append(not_count)
if not not_crash and not_args == ["not"]:
proc_not_fail_if_crash.append(True)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 4b3b45cf8fa2e..a82a589029ef7 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -529,6 +529,13 @@ def parse_args():
help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
action="store_true",
)
+ debug_group.add_argument(
+ "--use-process-workers",
+ dest="use_process_workers",
+ action="store_true",
+ default=False,
+ help="Run tests in separate worker processes instead of threads",
+ )
# LIT is special: environment variables override command line arguments.
env_args = shlex.split(os.environ.get("LIT_OPTS", ""))
diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py
index e567cce541beb..9e740be790914 100644
--- a/llvm/utils/lit/lit/formats/googletest.py
+++ b/llvm/utils/lit/lit/formats/googletest.py
@@ -171,7 +171,7 @@ def execute(self, test, litConfig):
"GTEST_SHARD_INDEX": os.environ.get("GTEST_SHARD_INDEX", shard_idx),
}
)
- test.config.environment.update(shard_env)
+ test_env = {**test.config.environment, **shard_env}
cmd = [testPath]
cmd = self.prepareCmd(cmd)
@@ -190,7 +190,7 @@ def get_shard_header(shard_env):
try:
out, _, exitCode = lit.util.executeCommand(
cmd,
- env=test.config.environment,
+ env=test_env,
timeout=test.config.maxIndividualTestTime,
redirect_stderr=True,
)
@@ -353,6 +353,7 @@ def remove_gtest(tests):
elapsed_time = float(testinfo["time"][:-1])
res = lit.Test.Result(returnCode, output, elapsed_time)
res.pid = test.result.pid or 0
+ res.tid = test.result.tid or 0
res.start = start_time
start_time = start_time + elapsed_time
subtest.setResult(res)
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index fb5e03fcede24..bdc9cff29c240 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -43,6 +43,7 @@ def main(builtin_params={}):
pass_env=opts.pass_env,
per_test_coverage=opts.per_test_coverage,
gtest_sharding=opts.gtest_sharding,
+ use_process_workers=opts.use_process_workers,
maxRetriesPerTest=opts.maxRetriesPerTest,
update_tests=opts.update_tests,
maxIndividualTestTime=opts.maxIndividualTestTime,
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index c5b09fe81bb21..364bab6aab7f6 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -294,9 +294,10 @@ def _get_test_event(self, test, first_start_time):
elapsed_time = test.result.elapsed or 0.0
start_time = test.result.start - first_start_time if test.result.start else 0.0
pid = test.result.pid or 0
+ tid = test.result.tid or 0
return {
"pid": pid,
- "tid": 1,
+ "tid": tid,
"ph": "X",
"ts": int(start_time * 1000000.0),
"dur": int(elapsed_time * 1000000.0),
diff --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py
index d1c56491cee08..0970294fb7973 100644
--- a/llvm/utils/lit/lit/run.py
+++ b/llvm/utils/lit/lit/run.py
@@ -1,9 +1,10 @@
+import threading
import multiprocessing
import os
import platform
import sys
import time
-from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait
+from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, wait
from concurrent.futures.process import BrokenProcessPool
import concurrent.futures.process
@@ -135,53 +136,74 @@ def _abort_executors(self, executors, future_to_test):
except Exception:
pass
+ def _abort_thread_executors(self, executors, future_to_test):
+ for future in list(future_to_test):
+ future.cancel()
+ for ex in executors:
+ if sys.version_info >= (3, 9):
+ ex.shutdown(wait=False, cancel_futures=True)
+ else:
+ ex.shutdown(wait=False)
+
def _execute(self, deadline):
+ use_threads = not self.lit_config.use_process_workers
+
self._increase_process_limit()
+ semaphore_class = (
+ threading.BoundedSemaphore if use_threads else multiprocessing.BoundedSemaphore
+ )
semaphores = {
- k: multiprocessing.BoundedSemaphore(v)
+ k: semaphore_class(v)
for k, v in self.lit_config.parallelism_groups.items()
if v is not None
}
- # Windows has a limit of 60 workers per pool, so we need to use multiple pools
- # if we have more workers requested than the limit.
- # Also, allow to override the limit with the LIT_WINDOWS_MAX_WORKERS_PER_POOL environment variable.
- max_workers_per_pool = (
- WINDOWS_MAX_WORKERS_PER_POOL if os.name == "nt" else self.workers
- )
- max_workers_per_pool = int(
- os.getenv("LIT_WINDOWS_MAX_WORKERS_PER_POOL", max_workers_per_pool)
- )
+ if use_threads:
+ lit.worker.initialize(self.lit_config, semaphores, ignore_sigint=False)
+ executors = [ThreadPoolExecutor(max_workers=self.workers)]
+ else:
+ # Windows has a limit of 60 workers per pool, so we need to use multiple pools
+ # if we have more workers requested than the limit.
+ # Also, allow to override the limit with the LIT_WINDOWS_MAX_WORKERS_PER_POOL environment variable.
+ max_workers_per_pool = (
+ WINDOWS_MAX_WORKERS_PER_POOL if os.name == "nt" else self.workers
+ )
+ max_workers_per_pool = int(
+ os.getenv("LIT_WINDOWS_MAX_WORKERS_PER_POOL", max_workers_per_pool)
+ )
- num_pools = max(1, _ceilDiv(self.workers, max_workers_per_pool))
+ num_pools = max(1, _ceilDiv(self.workers, max_workers_per_pool))
- # Distribute self.workers across num_pools as evenly as possible
- workers_per_pool_list = [self.workers // num_pools] * num_pools
- for pool_idx in range(self.workers % num_pools):
- workers_per_pool_list[pool_idx] += 1
+ # Distribute self.workers across num_pools as evenly as possible
+ workers_per_pool_list = [self.workers // num_pools] * num_pools
+ for pool_idx in range(self.workers % num_pools):
+ workers_per_pool_list[pool_idx] += 1
- if num_pools > 1:
- self.lit_config.note(
- "Using %d pools balancing %d workers total distributed as %s (Windows worker limit workaround)"
- % (num_pools, self.workers, workers_per_pool_list)
- )
+ if num_pools > 1:
+ self.lit_config.note(
+ "Using %d pools balancing %d workers total distributed as %s (Windows worker limit workaround)"
+ % (num_pools, self.workers, workers_per_pool_list)
+ )
- executors = [
- ProcessPoolExecutor(
- max_workers=pool_size,
- initializer=lit.worker.initialize,
- initargs=(self.lit_config, semaphores),
- )
- for pool_size in workers_per_pool_list
- ]
+ executors = [
+ ProcessPoolExecutor(
+ max_workers=pool_size,
+ initializer=lit.worker.initialize,
+ initargs=(self.lit_config, semaphores),
+ )
+ for pool_size in workers_per_pool_list
+ ]
future_to_test = {}
try:
self._dispatch_and_wait(executors, future_to_test, deadline)
except BaseException:
- self._abort_executors(executors, future_to_test)
+ if use_threads:
+ self._abort_thread_executors(executors, future_to_test)
+ else:
+ self._abort_executors(executors, future_to_test)
raise
else:
for ex in executors:
diff --git a/llvm/utils/lit/lit/util.py b/llvm/utils/lit/lit/util.py
index 8ab109fe15d45..1a7d5fc74ef7b 100644
--- a/llvm/utils/lit/lit/util.py
+++ b/llvm/utils/lit/lit/util.py
@@ -447,15 +447,21 @@ def killProcessAndChildren(pid):
def memoize(f):
cache = {} # Unbounded
+ lock = threading.Lock()
def make_key(args, kwargs):
return args, tuple(kwargs.items())
def memoized(*args, **kwargs):
key = make_key(args, kwargs)
- if key not in cache:
- cache[key] = f(*args, **kwargs)
- return cache[key]
+ try:
+ return cache[key]
+ except KeyError:
+ pass
+ with lock:
+ if key not in cache:
+ cache[key] = f(*args, **kwargs)
+ return cache[key]
return memoized
diff --git a/llvm/utils/lit/lit/worker.py b/llvm/utils/lit/lit/worker.py
index 0e36f3e9a2424..f6448a4fd2668 100644
--- a/llvm/utils/lit/lit/worker.py
+++ b/llvm/utils/lit/lit/worker.py
@@ -8,6 +8,7 @@
import contextlib
import os
import signal
+import threading
import time
import traceback
@@ -20,7 +21,7 @@
_parallelism_semaphores = None
-def initialize(lit_config, parallelism_semaphores):
+def initialize(lit_config, parallelism_semaphores, ignore_sigint=True):
"""Copy data shared by all test executions into worker processes"""
global _lit_config
global _parallelism_semaphores
@@ -30,7 +31,8 @@ def initialize(lit_config, parallelism_semaphores):
# We use the following strategy for dealing with Ctrl+C/KeyboardInterrupt in
# subprocesses created by the multiprocessing.Pool.
# https://noswap.com/blog/python-multiprocessing-keyboardinterrupt
- signal.signal(signal.SIGINT, signal.SIG_IGN)
+ if ignore_sigint:
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
def execute(test):
@@ -69,6 +71,7 @@ def _execute(test, lit_config):
result.elapsed = time.time() - start
result.start = start
result.pid = os.getpid()
+ result.tid = threading.get_ident()
return result
More information about the llvm-commits
mailing list