[llvm] [lit] Migrate lit to ThreadPoolExecutor (PR #212397)

Prasoon Kumar via llvm-commits llvm-commits at lists.llvm.org
Mon Jul 27 20:48:26 PDT 2026


https://github.com/prasoon054 created https://github.com/llvm/llvm-project/pull/212397

Lit currently runs each test in its own worker process via ProcessPoolExecutor. Move the default execution engine to ThreadPoolExecutor instead: a worker process pays fork/exec plus pickling overhead per test that a single-process, multi-threaded model avoids, and it's groundwork for a possible asyncio backend later. ProcessPoolExecutor stays available behind --use-process-workers.

>From 8df7fec397941d9359aa6606d61b1a101b5c20c8 Mon Sep 17 00:00:00 2001
From: Prasoon Kumar <prasoonkumar054 at gmail.com>
Date: Tue, 28 Jul 2026 06:36:58 +0530
Subject: [PATCH] [lit] Migrate lit to ThreadPoolExecutor

Lit currently runs each test in its own worker process via
ProcessPoolExecutor. Move the default execution engine to
ThreadPoolExecutor instead: a worker process pays fork/exec plus
pickling overhead per test that a single-process, multi-threaded model
avoids, and it's groundwork for a possible asyncio backend later.
ProcessPoolExecutor stays available behind --use-process-workers.

Signed-off-by: Prasoon Kumar <prasoonkumar054 at gmail.com>
---
 llvm/utils/lit/lit/LitConfig.py          |  2 +
 llvm/utils/lit/lit/Test.py               |  1 +
 llvm/utils/lit/lit/TestRunner.py         | 64 +++++++++++++++---
 llvm/utils/lit/lit/cl_arguments.py       |  7 ++
 llvm/utils/lit/lit/formats/googletest.py |  5 +-
 llvm/utils/lit/lit/main.py               |  1 +
 llvm/utils/lit/lit/reports.py            |  3 +-
 llvm/utils/lit/lit/run.py                | 82 +++++++++++++++---------
 llvm/utils/lit/lit/util.py               | 12 +++-
 llvm/utils/lit/lit/worker.py             |  7 +-
 10 files changed, 138 insertions(+), 46 deletions(-)

diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index 1bc466014f93d..53655899eb437 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -41,6 +41,7 @@ def __init__(
         parallelism_groups={},
         per_test_coverage=False,
         gtest_sharding=True,
+        use_process_workers=False,
         update_tests=False,
     ):
         # The name of the test runner.
@@ -53,6 +54,7 @@ def __init__(
         self.valgrindUserArgs = list(valgrindArgs)
         self.noExecute = noExecute
         self.debug = debug
+        self.use_process_workers = bool(use_process_workers)
         self.isWindows = bool(isWindows)
         self.order = order
         self.params = dict(params)
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 35da7afc591c0..d58a43a214a21 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -170,6 +170,7 @@ def __init__(
         self.elapsed = elapsed
         self.start = None
         self.pid = None
+        self.tid = None
         # The metrics reported by this test.
         self.metrics = {}
         # The micro-test results reported by this test.
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 97cb54c63be64..550268014a620 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import enum
 import os
 import pathlib
@@ -70,6 +71,48 @@ def __init__(self, message):
 kPdbgRegex = "%dbg\\(([^)'\"]*)\\)((?:.|\\n)*)"
 
 
+class _UmaskGate:
+    def __init__(self):
+        self._cv = threading.Condition()
+        self._active_readers = 0
+        self._writer_active = False
+        self._writers_waiting = 0
+
+    @contextlib.contextmanager
+    def read(self):
+        with self._cv:
+            while self._writer_active or self._writers_waiting:
+                self._cv.wait()
+            self._active_readers += 1
+        try:
+            yield
+        finally:
+            with self._cv:
+                self._active_readers -= 1
+                if self._active_readers == 0:
+                    self._cv.notify_all()
+
+    @contextlib.contextmanager
+    def write(self):
+        with self._cv:
+            self._writers_waiting += 1
+            try:
+                while self._writer_active or self._active_readers > 0:
+                    self._cv.wait()
+            finally:
+                self._writers_waiting -= 1
+            self._writer_active = True
+        try:
+            yield
+        finally:
+            with self._cv:
+                self._writer_active = False
+                self._cv.notify_all()
+
+
+_umask_gate = _UmaskGate()
+
+
 def buildPdbgCommand(msg, cmd):
     res = f"%dbg({msg}) {cmd}"
     assert re.fullmatch(
@@ -463,11 +506,8 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
             # os.umask as the umask argument in subprocess.Popen is not
             # available before Python 3.9. Once LLVM requires at least Python
             # 3.9, this code should be updated to use umask argument.
-            old_umask = -1
-            if cmd_shenv.umask != -1:
-                old_umask = os.umask(cmd_shenv.umask)
-            procs.append(
-                subprocess.Popen(
+            def spawn():
+                return subprocess.Popen(
                     args,
                     cwd=cmd_shenv.cwd,
                     executable=executable,
@@ -479,9 +519,17 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
                     universal_newlines=True,
                     errors="replace",
                 )
-            )
-            if old_umask != -1:
-                os.umask(old_umask)
+
+            if cmd_shenv.umask != -1:
+                with _umask_gate.write():
+                    old_umask = os.umask(cmd_shenv.umask)
+                    try:
+                        procs.append(spawn())
+                    finally:
+                        os.umask(old_umask)
+            else:
+                with _umask_gate.read():
+                    procs.append(spawn())
             proc_not_counts.append(not_count)
             if not not_crash and not_args == ["not"]:
                 proc_not_fail_if_crash.append(True)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 4b3b45cf8fa2e..a82a589029ef7 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -529,6 +529,13 @@ def parse_args():
         help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
         action="store_true",
     )
+    debug_group.add_argument(
+        "--use-process-workers",
+        dest="use_process_workers",
+        action="store_true",
+        default=False,
+        help="Run tests in separate worker processes instead of threads",
+    )
 
     # LIT is special: environment variables override command line arguments.
     env_args = shlex.split(os.environ.get("LIT_OPTS", ""))
diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py
index e567cce541beb..9e740be790914 100644
--- a/llvm/utils/lit/lit/formats/googletest.py
+++ b/llvm/utils/lit/lit/formats/googletest.py
@@ -171,7 +171,7 @@ def execute(self, test, litConfig):
                     "GTEST_SHARD_INDEX": os.environ.get("GTEST_SHARD_INDEX", shard_idx),
                 }
             )
-        test.config.environment.update(shard_env)
+        test_env = {**test.config.environment, **shard_env}
 
         cmd = [testPath]
         cmd = self.prepareCmd(cmd)
@@ -190,7 +190,7 @@ def get_shard_header(shard_env):
         try:
             out, _, exitCode = lit.util.executeCommand(
                 cmd,
-                env=test.config.environment,
+                env=test_env,
                 timeout=test.config.maxIndividualTestTime,
                 redirect_stderr=True,
             )
@@ -353,6 +353,7 @@ def remove_gtest(tests):
                         elapsed_time = float(testinfo["time"][:-1])
                         res = lit.Test.Result(returnCode, output, elapsed_time)
                         res.pid = test.result.pid or 0
+                        res.tid = test.result.tid or 0
                         res.start = start_time
                         start_time = start_time + elapsed_time
                         subtest.setResult(res)
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index fb5e03fcede24..bdc9cff29c240 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -43,6 +43,7 @@ def main(builtin_params={}):
         pass_env=opts.pass_env,
         per_test_coverage=opts.per_test_coverage,
         gtest_sharding=opts.gtest_sharding,
+        use_process_workers=opts.use_process_workers,
         maxRetriesPerTest=opts.maxRetriesPerTest,
         update_tests=opts.update_tests,
         maxIndividualTestTime=opts.maxIndividualTestTime,
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index c5b09fe81bb21..364bab6aab7f6 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -294,9 +294,10 @@ def _get_test_event(self, test, first_start_time):
         elapsed_time = test.result.elapsed or 0.0
         start_time = test.result.start - first_start_time if test.result.start else 0.0
         pid = test.result.pid or 0
+        tid = test.result.tid or 0
         return {
             "pid": pid,
-            "tid": 1,
+            "tid": tid,
             "ph": "X",
             "ts": int(start_time * 1000000.0),
             "dur": int(elapsed_time * 1000000.0),
diff --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py
index d1c56491cee08..0970294fb7973 100644
--- a/llvm/utils/lit/lit/run.py
+++ b/llvm/utils/lit/lit/run.py
@@ -1,9 +1,10 @@
+import threading
 import multiprocessing
 import os
 import platform
 import sys
 import time
-from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait
+from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, wait
 from concurrent.futures.process import BrokenProcessPool
 import concurrent.futures.process
 
@@ -135,53 +136,74 @@ def _abort_executors(self, executors, future_to_test):
         except Exception:
             pass
 
+    def _abort_thread_executors(self, executors, future_to_test):
+        for future in list(future_to_test):
+            future.cancel()
+        for ex in executors:
+            if sys.version_info >= (3, 9):
+                ex.shutdown(wait=False, cancel_futures=True)
+            else:
+                ex.shutdown(wait=False)
+
     def _execute(self, deadline):
+        use_threads = not self.lit_config.use_process_workers
+
         self._increase_process_limit()
 
+        semaphore_class = (
+            threading.BoundedSemaphore if use_threads else multiprocessing.BoundedSemaphore
+        )
         semaphores = {
-            k: multiprocessing.BoundedSemaphore(v)
+            k: semaphore_class(v)
             for k, v in self.lit_config.parallelism_groups.items()
             if v is not None
         }
 
-        # Windows has a limit of 60 workers per pool, so we need to use multiple pools
-        # if we have more workers requested than the limit.
-        # Also, allow to override the limit with the LIT_WINDOWS_MAX_WORKERS_PER_POOL environment variable.
-        max_workers_per_pool = (
-            WINDOWS_MAX_WORKERS_PER_POOL if os.name == "nt" else self.workers
-        )
-        max_workers_per_pool = int(
-            os.getenv("LIT_WINDOWS_MAX_WORKERS_PER_POOL", max_workers_per_pool)
-        )
+        if use_threads:
+            lit.worker.initialize(self.lit_config, semaphores, ignore_sigint=False)
+            executors = [ThreadPoolExecutor(max_workers=self.workers)]
+        else:
+            # Windows has a limit of 60 workers per pool, so we need to use multiple pools
+            # if we have more workers requested than the limit.
+            # Also, allow to override the limit with the LIT_WINDOWS_MAX_WORKERS_PER_POOL environment variable.
+            max_workers_per_pool = (
+                WINDOWS_MAX_WORKERS_PER_POOL if os.name == "nt" else self.workers
+            )
+            max_workers_per_pool = int(
+                os.getenv("LIT_WINDOWS_MAX_WORKERS_PER_POOL", max_workers_per_pool)
+            )
 
-        num_pools = max(1, _ceilDiv(self.workers, max_workers_per_pool))
+            num_pools = max(1, _ceilDiv(self.workers, max_workers_per_pool))
 
-        # Distribute self.workers across num_pools as evenly as possible
-        workers_per_pool_list = [self.workers // num_pools] * num_pools
-        for pool_idx in range(self.workers % num_pools):
-            workers_per_pool_list[pool_idx] += 1
+            # Distribute self.workers across num_pools as evenly as possible
+            workers_per_pool_list = [self.workers // num_pools] * num_pools
+            for pool_idx in range(self.workers % num_pools):
+                workers_per_pool_list[pool_idx] += 1
 
-        if num_pools > 1:
-            self.lit_config.note(
-                "Using %d pools balancing %d workers total distributed as %s (Windows worker limit workaround)"
-                % (num_pools, self.workers, workers_per_pool_list)
-            )
+            if num_pools > 1:
+                self.lit_config.note(
+                    "Using %d pools balancing %d workers total distributed as %s (Windows worker limit workaround)"
+                    % (num_pools, self.workers, workers_per_pool_list)
+                )
 
-        executors = [
-            ProcessPoolExecutor(
-                max_workers=pool_size,
-                initializer=lit.worker.initialize,
-                initargs=(self.lit_config, semaphores),
-            )
-            for pool_size in workers_per_pool_list
-        ]
+            executors = [
+                ProcessPoolExecutor(
+                    max_workers=pool_size,
+                    initializer=lit.worker.initialize,
+                    initargs=(self.lit_config, semaphores),
+                )
+                for pool_size in workers_per_pool_list
+            ]
 
         future_to_test = {}
 
         try:
             self._dispatch_and_wait(executors, future_to_test, deadline)
         except BaseException:
-            self._abort_executors(executors, future_to_test)
+            if use_threads:
+                self._abort_thread_executors(executors, future_to_test)
+            else:
+                self._abort_executors(executors, future_to_test)
             raise
         else:
             for ex in executors:
diff --git a/llvm/utils/lit/lit/util.py b/llvm/utils/lit/lit/util.py
index 8ab109fe15d45..1a7d5fc74ef7b 100644
--- a/llvm/utils/lit/lit/util.py
+++ b/llvm/utils/lit/lit/util.py
@@ -447,15 +447,21 @@ def killProcessAndChildren(pid):
 
 def memoize(f):
     cache = {}  # Unbounded
+    lock = threading.Lock()
 
     def make_key(args, kwargs):
         return args, tuple(kwargs.items())
 
     def memoized(*args, **kwargs):
         key = make_key(args, kwargs)
-        if key not in cache:
-            cache[key] = f(*args, **kwargs)
-        return cache[key]
+        try:
+            return cache[key]
+        except KeyError:
+            pass
+        with lock:
+            if key not in cache:
+                cache[key] = f(*args, **kwargs)
+            return cache[key]
 
     return memoized
 
diff --git a/llvm/utils/lit/lit/worker.py b/llvm/utils/lit/lit/worker.py
index 0e36f3e9a2424..f6448a4fd2668 100644
--- a/llvm/utils/lit/lit/worker.py
+++ b/llvm/utils/lit/lit/worker.py
@@ -8,6 +8,7 @@
 import contextlib
 import os
 import signal
+import threading
 import time
 import traceback
 
@@ -20,7 +21,7 @@
 _parallelism_semaphores = None
 
 
-def initialize(lit_config, parallelism_semaphores):
+def initialize(lit_config, parallelism_semaphores, ignore_sigint=True):
     """Copy data shared by all test executions into worker processes"""
     global _lit_config
     global _parallelism_semaphores
@@ -30,7 +31,8 @@ def initialize(lit_config, parallelism_semaphores):
     # We use the following strategy for dealing with Ctrl+C/KeyboardInterrupt in
     # subprocesses created by the multiprocessing.Pool.
     # https://noswap.com/blog/python-multiprocessing-keyboardinterrupt
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    if ignore_sigint:
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 
 def execute(test):
@@ -69,6 +71,7 @@ def _execute(test, lit_config):
     result.elapsed = time.time() - start
     result.start = start
     result.pid = os.getpid()
+    result.tid = threading.get_ident()
     return result
 
 



More information about the llvm-commits mailing list