[llvm] [Lit] Enable users to supply custom in-process built-ins via the Lit config #16 (PR #195114)
Benjamin Stott via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 08:56:44 PDT 2026
https://github.com/BStott6 created https://github.com/llvm/llvm-project/pull/195114
This PR enables Lit users to supply new in-process built-ins via the constructor of lit.formats.ShTest. This will be used by the LLVM Lit configs to supply the in-process built-ins responsible for invoking the daemon to Lit. I've also added a new test case which registers some custom in-process built-ins and verifies they work as expected.
This PR is blocked on https://github.com/llvm/llvm-project/pull/194664; the first two commits in this PR are from that one, so please review only the third commit on this PR. Once that PR is merged, I will rebase this branch to remove the other commits.
This PR is part of a series of patches upgrading Lit's in-process built-ins to be able to run with piped input/output and full redirection support, and to allow custom in-process builtns to be provided via the Lit config. The remaining patches to Lit's test runner can be found [here](https://github.com/BStott6/llvm-project/compare/lit-inproc-builtins). This is part of the [Lit daemonized testing project](https://discourse.llvm.org/t/rfc-reducing-process-creation-overhead-in-llvm-regression-tests/88612/9).
>From 3068a4dcb50451506624b65511f8994b6813a0a5 Mon Sep 17 00:00:00 2001
From: BStott <Benjamin.Stott at sony.com>
Date: Tue, 28 Apr 2026 17:05:04 +0100
Subject: [PATCH 1/3] Introduce a new IO interface for in-process builtins
---
llvm/utils/lit/lit/InprocBuiltins.py | 251 +++++++++++++++++++--------
llvm/utils/lit/lit/TestRunner.py | 54 ++++--
llvm/utils/lit/tests/shtest-glob.py | 3 +-
llvm/utils/lit/tests/shtest-shell.py | 37 +---
4 files changed, 219 insertions(+), 126 deletions(-)
diff --git a/llvm/utils/lit/lit/InprocBuiltins.py b/llvm/utils/lit/lit/InprocBuiltins.py
index 34a4d7eae1e88..9bc43f4d30143 100644
--- a/llvm/utils/lit/lit/InprocBuiltins.py
+++ b/llvm/utils/lit/lit/InprocBuiltins.py
@@ -5,77 +5,158 @@
import shutil
import stat
import subprocess
-from io import BytesIO, StringIO
+from dataclasses import dataclass
+from io import BytesIO
+from typing import Any, Callable, Dict, List, Optional
import lit.util
+from lit.ShCommands import Command
from lit.ShellEnvironment import (
InternalShellError,
- ShellCommandResult,
- expand_glob_expressions,
+ ShellEnvironment,
kIsWindows,
- processRedirects,
updateEnv,
)
-def executeBuiltinCd(cmd, shenv):
+class InprocBuiltinIO:
+ """
+ Holds IO streams for an inproc builtin invocation.
+
+ NB: If stderr is redirected to be the same stream as stdout, then
+ `stder == stdout` is True.
+ """
+
+ stdin: Any
+ stdout: Any
+ stderr: Any
+
+ def __init__(self, stdin, stdout, stderr):
+ """
+ Configure the IO streams for an in-process built-in command. This
+ constructor is designed to mirror how IO streams are configured in
+ `subprocess.Popen`.
+
+ Each of the arguments to this constructor may be:
+ - A file object open in binary mode.
+ - `subprocess.PIPE` or `subprocess.STDOUT` sentinels.
+ - None.
+ """
+
+ # If stderr is redirected to stdout, we make sure to use the same
+ # stream for both so that the order of output is preserved.
+ stderr_redirected_to_stdout = (
+ stdout == subprocess.PIPE and stderr == subprocess.STDOUT
+ )
+
+ # Replace sentinel values with in-memory streams.
+ if stdin == subprocess.PIPE or stdin is None:
+ self.stdin = BytesIO()
+ else:
+ self.stdin = stdin
+
+ if stdout == subprocess.PIPE or stdout is None:
+ self.stdout = BytesIO()
+ else:
+ self.stdout = stdout
+
+ if stderr_redirected_to_stdout:
+ # Make sure stderr and stdout are directed to the same stream.
+ self.stderr = self.stdout
+ elif stderr == subprocess.PIPE or stderr is None:
+ self.stderr = BytesIO()
+ else:
+ self.stderr = stderr
+
+
+InprocBuiltinExecuteFn = Callable[
+ [Command, List[str], ShellEnvironment, InprocBuiltinIO],
+ int,
+]
+"""
+Function called by an in-process built-in command.
+Parameters:
+ - `cmd`: The command itself.
+ - `args`: glob-expanded list of arguments (including argv[0] as the program
+ name).
+ - `shenv`: The shell environment.
+ - `io`: Holds the input and output streams for the invocation. These are
+ binary IO objects (files open in binary mode, BytesIO).
+
+The return value is the exit code.
+"""
+
+
+ at dataclass
+class InprocBuiltin:
+ """
+ Represents a command that is run as an in-process built-in.
+ """
+
+ execute: InprocBuiltinExecuteFn
+ """
+ Function called to execute the in-process built-in.
+ """
+
+ fallback: Optional[str] = None
+
+
+def executeBuiltinCd(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+) -> int:
"""executeBuiltinCd - Change the current directory."""
- if len(cmd.args) != 2:
+ if len(args) != 2:
raise InternalShellError(cmd, "'cd' supports only one argument")
# Update the cwd in the parent environment.
- shenv.change_dir(cmd.args[1])
+ shenv.change_dir(args[1])
# The cd builtin always succeeds. If the directory does not exist, the
# following Popen calls will fail instead.
- return ShellCommandResult(cmd, "", "", 0, False)
+ return 0
-def executeBuiltinPushd(cmd, shenv):
+def executeBuiltinPushd(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+) -> int:
"""executeBuiltinPushd - Change the current dir and save the old."""
- if len(cmd.args) != 2:
+ if len(args) != 2:
raise InternalShellError(cmd, "'pushd' supports only one argument")
shenv.dirStack.append(shenv.cwd)
- shenv.change_dir(cmd.args[1])
- return ShellCommandResult(cmd, "", "", 0, False)
+ shenv.change_dir(args[1])
+ return 0
-def executeBuiltinPopd(cmd, shenv):
+def executeBuiltinPopd(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+) -> int:
"""executeBuiltinPopd - Restore a previously saved working directory."""
- if len(cmd.args) != 1:
+ if len(args) != 1:
raise InternalShellError(cmd, "'popd' does not support arguments")
if not shenv.dirStack:
raise InternalShellError(cmd, "popd: directory stack empty")
shenv.cwd = shenv.dirStack.pop()
- return ShellCommandResult(cmd, "", "", 0, False)
+ return 0
-def executeBuiltinExport(cmd, shenv):
+def executeBuiltinExport(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+) -> int:
"""executeBuiltinExport - Set an environment variable."""
- if len(cmd.args) != 2:
+ if len(args) != 2:
raise InternalShellError(cmd, "'export' supports only one argument")
- updateEnv(shenv, cmd.args)
- return ShellCommandResult(cmd, "", "", 0, False)
+ updateEnv(shenv, args)
+ return 0
-def executeBuiltinEcho(cmd, shenv):
+def executeBuiltinEcho(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+) -> int:
"""Interpret a redirected echo or @echo command"""
opened_files = []
- stdin, stdout, stderr = processRedirects(cmd, subprocess.PIPE, shenv, opened_files)
- if stdin != subprocess.PIPE or stderr != subprocess.PIPE:
- raise InternalShellError(
- cmd, f"stdin and stderr redirects not supported for {cmd.args[0]}"
- )
-
- # Some tests have un-redirected echo commands to help debug test failures.
- # Buffer our output and return it to the caller.
- is_redirected = True
- if stdout == subprocess.PIPE:
- is_redirected = False
- stdout = BytesIO()
# Implement echo flags. We only support -e and -n, and not yet in
# combination. We have to ignore unknown flags, because `echo "-D FOO"`
# prints the dash.
- args = cmd.args[1:]
+ args = args[1:]
interpret_escapes = False
write_newline = True
while len(args) >= 1 and args[0] in ("-e", "-n"):
@@ -94,30 +175,24 @@ def maybeUnescape(arg):
if args:
for arg in args[:-1]:
- stdout.write(maybeUnescape(arg).encode())
- stdout.write(b" ")
- stdout.write(maybeUnescape(args[-1]).encode())
+ io.stdout.write(maybeUnescape(arg).encode())
+ io.stdout.write(" ".encode())
+ io.stdout.write(maybeUnescape(args[-1]).encode())
if write_newline:
- stdout.write("\n".encode())
+ io.stdout.write("\n".encode())
for name, mode, f, path in opened_files:
f.close()
- output = (
- ""
- if is_redirected
- # TODO(BStott) remove decode once new interface for in-process builtin
- # IO is introduced.
- else stdout.getvalue().decode(encoding="utf8", errors="replace")
- )
- return ShellCommandResult(cmd, output, "", 0, False)
+ return 0
-def executeBuiltinMkdir(cmd, cmd_shenv):
+def executeBuiltinMkdir(
+ cmd: Command, args: List[str], cmd_shenv: ShellEnvironment, io: InprocBuiltinIO
+):
"""executeBuiltinMkdir - Create new directories."""
- args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
try:
- opts, args = getopt.gnu_getopt(args, "p")
+ opts, args = getopt.gnu_getopt(args[1:], "p")
except getopt.GetoptError as err:
raise InternalShellError(cmd, "Unsupported: 'mkdir': %s" % str(err))
@@ -131,7 +206,6 @@ def executeBuiltinMkdir(cmd, cmd_shenv):
if len(args) == 0:
raise InternalShellError(cmd, "Error: 'mkdir' is missing an operand")
- stderr = StringIO()
exitCode = 0
for dir in args:
dir = pathlib.Path(dir)
@@ -144,16 +218,19 @@ def executeBuiltinMkdir(cmd, cmd_shenv):
try:
dir.mkdir(exist_ok=True)
except OSError as err:
- stderr.write("Error: 'mkdir' command failed, %s\n" % str(err))
+ io.stderr.write(
+ ("Error: 'mkdir' command failed, %s\n" % str(err)).encode()
+ )
exitCode = 1
- return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False)
+ return exitCode
-def executeBuiltinRm(cmd, cmd_shenv):
+def executeBuiltinRm(
+ cmd: Command, args: List[str], cmd_shenv: ShellEnvironment, io: InprocBuiltinIO
+):
"""executeBuiltinRm - Removes (deletes) files or directories."""
- args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
try:
- opts, args = getopt.gnu_getopt(args, "frR", ["--recursive"])
+ opts, args = getopt.gnu_getopt(args[1:], "frR", ["--recursive"])
except getopt.GetoptError as err:
raise InternalShellError(cmd, "Unsupported: 'rm': %s" % str(err))
@@ -176,7 +253,6 @@ def on_rm_error(func, path, exc_info):
os.chmod(path, stat.S_IMODE(os.stat(path).st_mode) | stat.S_IWRITE)
os.remove(path)
- stderr = StringIO()
exitCode = 0
for path in args:
cwd = cmd_shenv.cwd
@@ -189,7 +265,7 @@ def on_rm_error(func, path, exc_info):
os.remove(path)
elif os.path.isdir(path):
if not recursive:
- stderr.write("Error: %s is a directory\n" % path)
+ io.stderr.write(("Error: %s is a directory\n" % path).encode())
exitCode = 1
if platform.system() == "Windows":
# NOTE: use ctypes to access `SHFileOperationsW` on Windows to
@@ -255,26 +331,28 @@ class SHFILEOPSTRUCTW(Structure):
os.chmod(path, stat.S_IMODE(os.stat(path).st_mode) | stat.S_IWRITE)
os.remove(path)
except OSError as err:
- stderr.write("Error: 'rm' command failed, %s" % str(err))
+ io.stderr.write(("Error: 'rm' command failed, %s" % str(err)).encode())
exitCode = 1
- return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False)
+ return exitCode
-def executeBuiltinUmask(cmd, shenv):
+def executeBuiltinUmask(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+):
"""executeBuiltinUmask - Change the current umask."""
if os.name != "posix":
raise InternalShellError(cmd, "'umask' not supported on this system")
- if len(cmd.args) != 2:
+ if len(args) != 2:
raise InternalShellError(cmd, "'umask' supports only one argument")
try:
# Update the umask in the parent environment.
- shenv.umask = int(cmd.args[1], 8)
+ shenv.umask = int(args[1], 8)
except ValueError as err:
raise InternalShellError(cmd, "Error: 'umask': %s" % str(err))
- return ShellCommandResult(cmd, "", "", 0, False)
+ return 0
-def executeBuiltinUlimit(cmd, shenv):
+def executeBuiltinUlimit(cmd: Command, args: List[str], shenv, io: InprocBuiltinIO):
"""executeBuiltinUlimit - Change the current limits."""
try:
# Try importing the resource module (available on POSIX systems) and
@@ -282,34 +360,55 @@ def executeBuiltinUlimit(cmd, shenv):
import resource
except ImportError:
raise InternalShellError(cmd, "'ulimit' not supported on this system")
- if len(cmd.args) != 3:
+ if len(args) != 3:
raise InternalShellError(cmd, "'ulimit' requires two arguments")
try:
- if cmd.args[2] == "unlimited":
+ if args[2] == "unlimited":
new_limit = resource.RLIM_INFINITY
else:
- new_limit = int(cmd.args[2])
+ new_limit = int(args[2])
except ValueError as err:
raise InternalShellError(cmd, "Error: 'ulimit': %s" % str(err))
- if cmd.args[1] == "-v":
+ if args[1] == "-v":
if new_limit != resource.RLIM_INFINITY:
new_limit = new_limit * 1024
shenv.ulimit["RLIMIT_AS"] = new_limit
- elif cmd.args[1] == "-n":
+ elif args[1] == "-n":
shenv.ulimit["RLIMIT_NOFILE"] = new_limit
- elif cmd.args[1] == "-s":
+ elif args[1] == "-s":
if new_limit != resource.RLIM_INFINITY:
new_limit = new_limit * 1024
shenv.ulimit["RLIMIT_STACK"] = new_limit
- elif cmd.args[1] == "-f":
+ elif args[1] == "-f":
shenv.ulimit["RLIMIT_FSIZE"] = new_limit
else:
- raise InternalShellError(
- cmd, "'ulimit' does not support option: %s" % cmd.args[1]
- )
- return ShellCommandResult(cmd, "", "", 0, False)
+ raise InternalShellError(cmd, "'ulimit' does not support option: %s" % args[1])
+ return 0
-def executeBuiltinColon(cmd, cmd_shenv):
+def executeBuiltinColon(
+ cmd: Command, args: List[str], cmd_shenv: ShellEnvironment, io: InprocBuiltinIO
+):
"""executeBuiltinColon - Discard arguments and exit with status 0."""
- return ShellCommandResult(cmd, "", "", 0, False)
+ return 0
+
+
+def get_default_inproc_builtins() -> Dict[str, InprocBuiltin]:
+ """
+ get_default_inproc_builtins - Returns the map of command names to Lit's
+ in-process built-in implementations.
+ """
+
+ return {
+ "@echo": InprocBuiltin(executeBuiltinEcho, None),
+ "cd": InprocBuiltin(executeBuiltinCd, None),
+ "export": InprocBuiltin(executeBuiltinExport, None),
+ "echo": InprocBuiltin(executeBuiltinEcho, None),
+ "mkdir": InprocBuiltin(executeBuiltinMkdir, None),
+ "popd": InprocBuiltin(executeBuiltinPopd, None),
+ "pushd": InprocBuiltin(executeBuiltinPushd, None),
+ "rm": InprocBuiltin(executeBuiltinRm, None),
+ "ulimit": InprocBuiltin(executeBuiltinUlimit, None),
+ "umask": InprocBuiltin(executeBuiltinUmask, None),
+ ":": InprocBuiltin(executeBuiltinColon, None),
+ }
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index ee91342e969d9..09502c8e9e4c1 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -11,11 +11,11 @@
import threading
import traceback
-import lit.InprocBuiltins as InprocBuiltins
import lit.ShUtil as ShUtil
import lit.Test as Test
import lit.util
from lit.BooleanExpression import BooleanExpression
+from lit.InprocBuiltins import InprocBuiltinIO, get_default_inproc_builtins
from lit.ShCommands import Command
from lit.ShellEnvironment import (
InternalShellError,
@@ -254,19 +254,7 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
builtin_commands_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "builtin_commands"
)
- inproc_builtins = {
- "cd": InprocBuiltins.executeBuiltinCd,
- "export": InprocBuiltins.executeBuiltinExport,
- "echo": InprocBuiltins.executeBuiltinEcho,
- "@echo": InprocBuiltins.executeBuiltinEcho,
- "mkdir": InprocBuiltins.executeBuiltinMkdir,
- "popd": InprocBuiltins.executeBuiltinPopd,
- "pushd": InprocBuiltins.executeBuiltinPushd,
- "rm": InprocBuiltins.executeBuiltinRm,
- "ulimit": InprocBuiltins.executeBuiltinUlimit,
- "umask": InprocBuiltins.executeBuiltinUmask,
- ":": InprocBuiltins.executeBuiltinColon,
- }
+ inproc_builtins = get_default_inproc_builtins()
# To avoid deadlock, we use a single stderr stream for piped
# output. This is null until we have seen some output using
# stderr.
@@ -354,9 +342,43 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
j,
"Unsupported: '{}' cannot be part" " of a pipeline".format(args[0]),
)
- result = inproc_builtin(Command(args, j.redirects), cmd_shenv)
+
+ stdin, stdout, stderr = processRedirects(
+ j, subprocess.PIPE, shenv, opened_files
+ )
+
+ builtin_io = InprocBuiltinIO(stdin, stdout, stderr)
+
+ args = expand_glob_expressions(args, cmd_shenv.cwd)
+
+ exit_code = inproc_builtin.execute(
+ Command(args, j.redirects), args, cmd_shenv, builtin_io
+ )
+
+ builtin_io.stdout.flush()
+ builtin_io.stderr.flush()
+
if not_count % 2:
- result.exitCode = int(not result.exitCode)
+ exit_code = int(not exit_code)
+
+ # Gather output from the streams.
+ out = ""
+ if stdout == subprocess.PIPE:
+ builtin_io.stdout.seek(0)
+ out = builtin_io.stdout.read().decode("utf8", "replace")
+
+ err = ""
+ if stderr == subprocess.PIPE:
+ builtin_io.stderr.seek(0)
+ err = builtin_io.stderr.read().decode("utf8", "replace")
+
+ result = ShellCommandResult(
+ j,
+ out,
+ err,
+ exit_code,
+ False,
+ )
result.command.args = j.args
results.append(result)
return result.exitCode
diff --git a/llvm/utils/lit/tests/shtest-glob.py b/llvm/utils/lit/tests/shtest-glob.py
index 95a53e4dd2223..190745f8133d4 100644
--- a/llvm/utils/lit/tests/shtest-glob.py
+++ b/llvm/utils/lit/tests/shtest-glob.py
@@ -4,8 +4,7 @@
# RUN: | FileCheck -dump-input=fail -match-full-lines --implicit-check-not=Error: %s
# END.
-# CHECK: UNRESOLVED: shtest-glob :: glob-echo.txt ({{[^)]*}})
-# CHECK: AttributeError: 'GlobItem' object has no attribute 'encode'
+# CHECK: PASS: shtest-glob :: glob-echo.txt ({{[^)]*}})
# CHECK: FAIL: shtest-glob :: glob-mkdir.txt ({{[^)]*}})
# CHECK: # | Error: 'mkdir' command failed, {{.*}}example_file1.input'
diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py
index 498f6bb0adc11..fa5d63b5e9e29 100644
--- a/llvm/utils/lit/tests/shtest-shell.py
+++ b/llvm/utils/lit/tests/shtest-shell.py
@@ -497,37 +497,10 @@
# CHECK-NEXT: # error: command failed with exit status: 1
# CHECK: ***
-# CHECK: FAIL: shtest-shell :: echo-at-redirect-stderr.txt
-# CHECK: *** TEST 'shtest-shell :: echo-at-redirect-stderr.txt' FAILED ***
-# CHECK: @echo 2> {{.*}}
-# CHECK: # executed command: @echo
-# CHECK: # .---command stderr{{-*}}
-# CHECK: # | stdin and stderr redirects not supported for @echo
-# CHECK: error: command failed with exit status:
-
-# CHECK: FAIL: shtest-shell :: echo-at-redirect-stdin.txt
-# CHECK: *** TEST 'shtest-shell :: echo-at-redirect-stdin.txt' FAILED ***
-# CHECK: @echo < {{.*}}
-# CHECK: # executed command: @echo
-# CHECK: # .---command stderr{{-*}}
-# CHECK: # | stdin and stderr redirects not supported for @echo
-# CHECK: error: command failed with exit status:
-
-# CHECK: FAIL: shtest-shell :: echo-redirect-stderr.txt
-# CHECK: *** TEST 'shtest-shell :: echo-redirect-stderr.txt' FAILED ***
-# CHECK: echo 2> {{.*}}
-# CHECK: # executed command: echo
-# CHECK: # .---command stderr{{-*}}
-# CHECK: # | stdin and stderr redirects not supported for echo
-# CHECK: error: command failed with exit status:
-
-# CHECK: FAIL: shtest-shell :: echo-redirect-stdin.txt
-# CHECK: *** TEST 'shtest-shell :: echo-redirect-stdin.txt' FAILED ***
-# CHECK: echo < {{.*}}
-# CHECK: # executed command: echo
-# CHECK: # .---command stderr{{-*}}
-# CHECK: # | stdin and stderr redirects not supported for echo
-# CHECK: error: command failed with exit status:
+# CHECK: PASS: shtest-shell :: echo-at-redirect-stderr.txt
+# CHECK: PASS: shtest-shell :: echo-at-redirect-stdin.txt
+# CHECK: PASS: shtest-shell :: echo-redirect-stderr.txt
+# CHECK: PASS: shtest-shell :: echo-redirect-stdin.txt
# CHECK: FAIL: shtest-shell :: error-0.txt
# CHECK: *** TEST 'shtest-shell :: error-0.txt' FAILED ***
@@ -634,4 +607,4 @@
# CHECK: PASS: shtest-shell :: valid-shell.txt
# CHECK: Unresolved Tests (1)
-# CHECK: Failed Tests (37)
+# CHECK: Failed Tests (33)
>From e0644fee3c32f3b0c7f5b473b3f8b539932bd1b3 Mon Sep 17 00:00:00 2001
From: BStott <Benjamin.Stott at sony.com>
Date: Wed, 29 Apr 2026 11:16:39 +0100
Subject: [PATCH 2/3] Add typing.BinaryIO type hints for IO objects
---
llvm/utils/lit/lit/InprocBuiltins.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/llvm/utils/lit/lit/InprocBuiltins.py b/llvm/utils/lit/lit/InprocBuiltins.py
index 9bc43f4d30143..855694d5c4264 100644
--- a/llvm/utils/lit/lit/InprocBuiltins.py
+++ b/llvm/utils/lit/lit/InprocBuiltins.py
@@ -5,9 +5,10 @@
import shutil
import stat
import subprocess
+import typing
from dataclasses import dataclass
from io import BytesIO
-from typing import Any, Callable, Dict, List, Optional
+from typing import Callable, Dict, List, Optional
import lit.util
from lit.ShCommands import Command
@@ -27,9 +28,9 @@ class InprocBuiltinIO:
`stder == stdout` is True.
"""
- stdin: Any
- stdout: Any
- stderr: Any
+ stdin: typing.BinaryIO
+ stdout: typing.BinaryIO
+ stderr: typing.BinaryIO
def __init__(self, stdin, stdout, stderr):
"""
>From 5b468632ceeb2163c41d74f53a5736f154d10ab2 Mon Sep 17 00:00:00 2001
From: BStott <Benjamin.Stott at sony.com>
Date: Wed, 14 Jan 2026 18:15:59 +0000
Subject: [PATCH 3/3] Enable users to supply custom in-process builtins in
lit.cfg
---
llvm/utils/lit/lit/TestRunner.py | 74 +++++++++++++++----
llvm/utils/lit/lit/formats/shtest.py | 8 +-
.../custom_inproc_builtins.py | 30 ++++++++
.../shtest-custom-inproc-builtins/lit.cfg | 23 ++++++
.../use-custom-inproc-builtins.txt | 13 ++++
.../tests/shtest-custom-inproc-builtins.py | 10 +++
6 files changed, 142 insertions(+), 16 deletions(-)
create mode 100644 llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/custom_inproc_builtins.py
create mode 100644 llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/use-custom-inproc-builtins.txt
create mode 100644 llvm/utils/lit/tests/shtest-custom-inproc-builtins.py
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 09502c8e9e4c1..0f47e48604541 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -159,7 +159,7 @@ def _kill(self):
self._doneKillPass = True
-def executeShCmd(cmd, shenv, results, timeout=0):
+def executeShCmd(cmd, shenv, results, timeout=0, extra_inproc_builtins={}):
"""
Wrapper around _executeShCmd that handles
timeout
@@ -170,7 +170,9 @@ def executeShCmd(cmd, shenv, results, timeout=0):
if timeout > 0:
timeoutHelper.startTimer()
try:
- finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper)
+ finalExitCode = _executeShCmd(
+ cmd, shenv, results, timeoutHelper, extra_inproc_builtins
+ )
except InternalShellError:
e = sys.exc_info()[1]
finalExitCode = 127
@@ -211,7 +213,7 @@ def _replaceReadFile(match):
return arguments
-def _executeShCmd(cmd, shenv, results, timeoutHelper):
+def _executeShCmd(cmd, shenv, results, timeoutHelper, extra_inproc_builtins):
if timeoutHelper.timeoutReached():
# Prevent further recursion if the timeout has been hit
# as we should try avoid launching more processes.
@@ -219,25 +221,37 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
if isinstance(cmd, ShUtil.Seq):
if cmd.op == ";":
- res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper)
- return _executeShCmd(cmd.rhs, shenv, results, timeoutHelper)
+ res = _executeShCmd(
+ cmd.lhs, shenv, results, timeoutHelper, extra_inproc_builtins
+ )
+ return _executeShCmd(
+ cmd.rhs, shenv, results, timeoutHelper, extra_inproc_builtins
+ )
if cmd.op == "&":
raise InternalShellError(cmd, "unsupported shell operator: '&'")
if cmd.op == "||":
- res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper)
+ res = _executeShCmd(
+ cmd.lhs, shenv, results, timeoutHelper, extra_inproc_builtins
+ )
if res != 0:
- res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper)
+ res = _executeShCmd(
+ cmd.rhs, shenv, results, timeoutHelper, extra_inproc_builtins
+ )
return res
if cmd.op == "&&":
- res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper)
+ res = _executeShCmd(
+ cmd.lhs, shenv, results, timeoutHelper, extra_inproc_builtins
+ )
if res is None:
return res
if res == 0:
- res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper)
+ res = _executeShCmd(
+ cmd.rhs, shenv, results, timeoutHelper, extra_inproc_builtins
+ )
return res
raise ValueError("Unknown shell command: %r" % cmd.op)
@@ -255,6 +269,7 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
os.path.dirname(os.path.abspath(__file__)), "builtin_commands"
)
inproc_builtins = get_default_inproc_builtins()
+ inproc_builtins.update(extra_inproc_builtins)
# To avoid deadlock, we use a single stderr stream for piped
# output. This is null until we have seen some output using
# stderr.
@@ -680,7 +695,13 @@ def formatOutput(title, data, limit=None):
# function), out contains only stdout from the script, err contains only stderr
# from the script, and there is no execution trace.
def executeScriptInternal(
- test, litConfig, tmpBase, commands, cwd, debug=True
+ test,
+ litConfig,
+ tmpBase,
+ commands,
+ cwd,
+ debug=True,
+ extra_inproc_builtins={},
) -> tuple[str, str, int, str | None, str | None]:
cmds = []
update_output = None
@@ -725,7 +746,11 @@ def executeScriptInternal(
shenv.env["LIT_CURRENT_TESTCASE"] = test.getFullName()
exitCode, timeoutInfo = executeShCmd(
- cmd, shenv, results, timeout=litConfig.maxIndividualTestTime
+ cmd,
+ shenv,
+ results,
+ timeout=litConfig.maxIndividualTestTime,
+ extra_inproc_builtins=extra_inproc_builtins,
)
out = err = ""
@@ -1843,7 +1868,14 @@ def parseIntegratedTestScript(test, additional_parsers=[], require_script=True):
return script
-def _runShTest(test, litConfig, useExternalSh, script, tmpBase) -> lit.Test.Result:
+def _runShTest(
+ test,
+ litConfig,
+ useExternalSh,
+ script,
+ tmpBase,
+ extra_inproc_builtins,
+) -> lit.Test.Result:
# Always returns the tuple (out, err, exitCode, timeoutInfo, status).
def runOnce(
execdir,
@@ -1879,7 +1911,12 @@ def runOnce(
res = executeScript(test, litConfig, tmpBase, scriptCopy, execdir)
else:
res = executeScriptInternal(
- test, litConfig, tmpBase, scriptCopy, execdir
+ test,
+ litConfig,
+ tmpBase,
+ scriptCopy,
+ execdir,
+ extra_inproc_builtins=extra_inproc_builtins,
)
except ScriptFatal as e:
out = f"# " + "\n# ".join(str(e).splitlines()) + "\n"
@@ -1956,7 +1993,12 @@ def _replaceReadFile(match):
return commandLine
def executeShTest(
- test, litConfig, useExternalSh, extra_substitutions=[], preamble_commands=[]
+ test,
+ litConfig,
+ useExternalSh,
+ extra_substitutions=[],
+ preamble_commands=[],
+ extra_inproc_builtins={},
):
if test.config.unsupported:
return lit.Test.Result(Test.UNSUPPORTED, "Test is unsupported")
@@ -1993,4 +2035,6 @@ def executeShTest(
for index, command in enumerate(script):
script[index] = _expandLateSubstitutionsExternal(command)
- return _runShTest(test, litConfig, useExternalSh, script, tmpBase)
+ return _runShTest(
+ test, litConfig, useExternalSh, script, tmpBase, extra_inproc_builtins
+ )
diff --git a/llvm/utils/lit/lit/formats/shtest.py b/llvm/utils/lit/lit/formats/shtest.py
index b4dc6f9ed32e2..0027f997de603 100644
--- a/llvm/utils/lit/lit/formats/shtest.py
+++ b/llvm/utils/lit/lit/formats/shtest.py
@@ -19,11 +19,16 @@ class ShTest(FileBasedTest):
"""
def __init__(
- self, execute_external=False, extra_substitutions=[], preamble_commands=[]
+ self,
+ execute_external=False,
+ extra_substitutions=[],
+ preamble_commands=[],
+ extra_inproc_builtins={},
):
self.execute_external = execute_external
self.extra_substitutions = extra_substitutions
self.preamble_commands = preamble_commands
+ self.extra_inproc_builtins = extra_inproc_builtins
def execute(self, test, litConfig):
return lit.TestRunner.executeShTest(
@@ -32,4 +37,5 @@ def execute(self, test, litConfig):
self.execute_external,
self.extra_substitutions,
self.preamble_commands,
+ self.extra_inproc_builtins,
)
diff --git a/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/custom_inproc_builtins.py b/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/custom_inproc_builtins.py
new file mode 100644
index 0000000000000..802f4bd42d8b0
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/custom_inproc_builtins.py
@@ -0,0 +1,30 @@
+from typing import List
+from lit.InprocBuiltins import InprocBuiltinIO
+from lit.ShCommands import Command
+from lit.ShellEnvironment import ShellEnvironment
+
+
+def returns_0(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+):
+ return 0
+
+
+def returns_1(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+):
+ return 1
+
+
+def custom_echo(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+):
+ io.stdout.write(args[1].encode())
+ return 0
+
+
+def echo_to_stderr(
+ cmd: Command, args: List[str], shenv: ShellEnvironment, io: InprocBuiltinIO
+):
+ io.stderr.write(args[1].encode())
+ return 0
diff --git a/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/lit.cfg
new file mode 100644
index 0000000000000..2bec9562c7a9f
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/lit.cfg
@@ -0,0 +1,23 @@
+import os
+import sys
+import lit.formats
+from lit.InprocBuiltins import InprocBuiltin
+
+# Import the module containing the custom in-process builtins.
+# NB: These cannot be defined in the lit.cfg module.
+sys.path.append(os.path.dirname(__file__))
+import custom_inproc_builtins
+
+config.name = "shtest-custom-inproc-builtins"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest(
+ extra_inproc_builtins={
+ "returns_0": InprocBuiltin(custom_inproc_builtins.returns_0),
+ "returns_1": InprocBuiltin(custom_inproc_builtins.returns_1),
+ "custom_echo": InprocBuiltin(custom_inproc_builtins.custom_echo),
+ "echo_to_stderr": InprocBuiltin(custom_inproc_builtins.echo_to_stderr),
+ }
+)
+config.test_source_root = None
+config.test_exec_root = None
+
diff --git a/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/use-custom-inproc-builtins.txt b/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/use-custom-inproc-builtins.txt
new file mode 100644
index 0000000000000..ea02f994f60cc
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-custom-inproc-builtins/use-custom-inproc-builtins.txt
@@ -0,0 +1,13 @@
+RUN: returns_0
+RUN: not returns_1
+
+RUN: custom_echo "hello" > %t1
+RUN: grep -x "hello" < %t1
+
+RUN: custom_echo ", world" >> %t1
+RUN: grep -x "hello, world" < %t1
+
+RUN: echo_to_stderr "goodbye" 2> %t2
+RUN: grep -x "goodbye" < %t2
+RUN: echo_to_stderr ", for now" 2>> %t2
+RUN: grep -x "goodbye, for now" < %t2
diff --git a/llvm/utils/lit/tests/shtest-custom-inproc-builtins.py b/llvm/utils/lit/tests/shtest-custom-inproc-builtins.py
new file mode 100644
index 0000000000000..c46d41849eaa9
--- /dev/null
+++ b/llvm/utils/lit/tests/shtest-custom-inproc-builtins.py
@@ -0,0 +1,10 @@
+## This test provides some custom in-process built-ins via the lit.cfg, and
+## verifies that their output can be redirected correctly.
+#
+# RUN: %{lit} -v %{inputs}/shtest-custom-inproc-builtins \
+# RUN: | FileCheck -match-full-lines %s
+# END.
+
+# CHECK: PASS: shtest-custom-inproc-builtins :: use-custom-inproc-builtins.txt ({{[^)]*}})
+
+# CHECK: Passed: 1 ({{[^)]*}})
More information about the llvm-commits
mailing list