[llvm] d769ce2 - [Lit] Change processRedirects to open all files in binary mode (#194368)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 28 07:50:10 PDT 2026
Author: Benjamin Stott
Date: 2026-04-28T15:50:05+01:00
New Revision: d769ce21768c66a04b3e46d1b330fd29238b34a5
URL: https://github.com/llvm/llvm-project/commit/d769ce21768c66a04b3e46d1b330fd29238b34a5
DIFF: https://github.com/llvm/llvm-project/commit/d769ce21768c66a04b3e46d1b330fd29238b34a5.diff
LOG: [Lit] Change processRedirects to open all files in binary mode (#194368)
This PR is the second in a series of patches upgrading Lit's in-process
built-ins to be able to run with piped input/output and full redirection
support, and to allow custom in-process builtns to be provided via the
Lit config. The remaining patches to Lit's test runner can be found here@
https://github.com/BStott6/llvm-project/compare/lit-inproc-builtins.
This is part of the Lit daemonized testing project:
https://discourse.llvm.org/t/88612.
This PR makes Lit's `processRedirects` function open all input/output
files in binary mode. This makes sure that in-process builtins have the
expected behaviour when reading and writing from them:
Newline translation is not required for any of the current in-process
built-ins, in fact, the in-process built-in for `echo`, which is the
only one that writes to `stdout`, explicitly re-opens the output file
with `newline=""` on Windows, to avoid newline translation. Also,
in-process builtins will eventually need to be able to read or write
binary data: for example, `opt` without `-S` running in daemon mode.
I believe this has no functional change for regular process invocations;
I have confirmed that programs invoked by Lit which write to files
opened in binary mode by Lit still have the newline translation
performed as normal on Windows, unless they change the mode of their
output stream themselves.
Added:
Modified:
llvm/utils/lit/lit/InprocBuiltins.py
llvm/utils/lit/lit/ShellEnvironment.py
llvm/utils/lit/lit/TestRunner.py
llvm/utils/lit/tests/shtest-glob.py
Removed:
################################################################################
diff --git a/llvm/utils/lit/lit/InprocBuiltins.py b/llvm/utils/lit/lit/InprocBuiltins.py
index b20aeb6993726..34a4d7eae1e88 100644
--- a/llvm/utils/lit/lit/InprocBuiltins.py
+++ b/llvm/utils/lit/lit/InprocBuiltins.py
@@ -5,7 +5,7 @@
import shutil
import stat
import subprocess
-from io import StringIO
+from io import BytesIO, StringIO
import lit.util
from lit.ShellEnvironment import (
@@ -70,13 +70,7 @@ def executeBuiltinEcho(cmd, shenv):
is_redirected = True
if stdout == subprocess.PIPE:
is_redirected = False
- stdout = StringIO()
- elif kIsWindows:
- # Reopen stdout with `newline=""` to avoid CRLF translation.
- # The versions of echo we are replacing on Windows all emit plain LF,
- # and the LLVM tests now depend on this.
- stdout = open(stdout.name, stdout.mode, encoding="utf-8", newline="")
- opened_files.append((None, None, stdout, None))
+ stdout = BytesIO()
# Implement echo flags. We only support -e and -n, and not yet in
# combination. We have to ignore unknown flags, because `echo "-D FOO"`
@@ -100,16 +94,22 @@ def maybeUnescape(arg):
if args:
for arg in args[:-1]:
- stdout.write(maybeUnescape(arg))
- stdout.write(" ")
- stdout.write(maybeUnescape(args[-1]))
+ stdout.write(maybeUnescape(arg).encode())
+ stdout.write(b" ")
+ stdout.write(maybeUnescape(args[-1]).encode())
if write_newline:
- stdout.write("\n")
+ stdout.write("\n".encode())
for name, mode, f, path in opened_files:
f.close()
- output = "" if is_redirected else stdout.getvalue()
+ output = (
+ ""
+ if is_redirected
+ # TODO(BStott) remove decode once new interface for in-process builtin
+ # IO is introduced.
+ else stdout.getvalue().decode(encoding="utf8", errors="replace")
+ )
return ShellCommandResult(cmd, output, "", 0, False)
diff --git a/llvm/utils/lit/lit/ShellEnvironment.py b/llvm/utils/lit/lit/ShellEnvironment.py
index 4f6753bd11b52..b4ff7b1a120f6 100644
--- a/llvm/utils/lit/lit/ShellEnvironment.py
+++ b/llvm/utils/lit/lit/ShellEnvironment.py
@@ -108,19 +108,19 @@ def processRedirects(cmd, stdin_source, cmd_shenv, opened_files):
redirects = [(0,), (1,), (2,)]
for op, filename in cmd.redirects:
if op == (">", 2):
- redirects[2] = [filename, "w", None]
+ redirects[2] = [filename, "wb", None]
elif op == (">>", 2):
- redirects[2] = [filename, "a", None]
+ redirects[2] = [filename, "ab", None]
elif op == (">&", 2) and filename in "012":
redirects[2] = redirects[int(filename)]
elif op == (">&",) or op == ("&>",):
- redirects[1] = redirects[2] = [filename, "w", None]
+ redirects[1] = redirects[2] = [filename, "wb", None]
elif op == (">",):
- redirects[1] = [filename, "w", None]
+ redirects[1] = [filename, "wb", None]
elif op == (">>",):
- redirects[1] = [filename, "a", None]
+ redirects[1] = [filename, "ab", None]
elif op == ("<",):
- redirects[0] = [filename, "r", None]
+ redirects[0] = [filename, "rb", None]
else:
raise InternalShellError(
cmd, "Unsupported redirect: %r" % ((op, filename),)
@@ -173,11 +173,12 @@ def processRedirects(cmd, stdin_source, cmd_shenv, opened_files):
else:
# Make sure relative paths are relative to the cwd.
redir_filename = os.path.join(cmd_shenv.cwd, name)
- fd = open(redir_filename, mode, encoding="utf-8")
+ fd = open(redir_filename, mode)
+
# Workaround a Win32 and/or subprocess bug when appending.
#
# FIXME: Actually, this is probably an instance of PR6753.
- if mode == "a":
+ if mode == "ab":
fd.seek(0, 2)
# Mutate the underlying redirect list so that we can redirect stdout
# and stderr to the same place without opening the file twice.
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 0842c58a5f88a..ee91342e969d9 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -521,11 +521,11 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
if procs[i].stdout is not None:
out = procs[i].stdout.read()
else:
- out = ""
+ out = b""
if procs[i].stderr is not None:
err = procs[i].stderr.read()
else:
- err = ""
+ err = b""
procData[i] = (out, err)
# Read stderr out of the temp files.
@@ -568,7 +568,7 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
output_files = []
if res != 0:
for (name, mode, f, path) in sorted(opened_files):
- if path is not None and mode in ("w", "a"):
+ if path is not None and mode in ("wb", "ab"):
try:
with open(path, "rb") as f:
data = f.read()
diff --git a/llvm/utils/lit/tests/shtest-glob.py b/llvm/utils/lit/tests/shtest-glob.py
index ba609e036c166..95a53e4dd2223 100644
--- a/llvm/utils/lit/tests/shtest-glob.py
+++ b/llvm/utils/lit/tests/shtest-glob.py
@@ -5,7 +5,7 @@
# END.
# CHECK: UNRESOLVED: shtest-glob :: glob-echo.txt ({{[^)]*}})
-# CHECK: TypeError: string argument expected, got 'GlobItem'
+# CHECK: AttributeError: 'GlobItem' object has no attribute 'encode'
# CHECK: FAIL: shtest-glob :: glob-mkdir.txt ({{[^)]*}})
# CHECK: # | Error: 'mkdir' command failed, {{.*}}example_file1.input'
More information about the llvm-commits
mailing list