[libcxx] [llvm] [utils] update how auto-updated tests are displayed when the test is retried (PR #181097)
Henrik G. Olsson via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 12 23:14:32 PST 2026
https://github.com/hnrklssn updated https://github.com/llvm/llvm-project/pull/181097
>From 8c7e7cd40c9cd12a0d37a1a3d4f115fba7c32d7a Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 11 Feb 2026 21:38:47 -0800
Subject: [PATCH 01/16] [utils] show output from all test updates for tests
with retries
This moves the test updater output from the test output to its own
section, to provide clearer info when tests are retried. Updates with
retries had two UX problems:
- if a test is retried and fails multiple times, only the last test
output is shown, but the test may be updated on each failure.
- if a test fails, is updated, and then passes, the test output is not
shown unless the `--test-output all` option is set.
Even with test updater output hoisted out from the test output, it is
still only printed when the test result is printed. If
`--print-result-after failed` is set, and the test passes after being
updated, it is marked FLAKYPASS and not displayed. This adds the option
`--print-result-after failed-or-flaky` which prints results for both
failing tests as well as FLAKYPASS tests.
---
llvm/utils/lit/lit/Test.py | 10 ++-
llvm/utils/lit/lit/TestRunner.py | 33 +++++----
llvm/utils/lit/lit/cl_arguments.py | 9 ++-
llvm/utils/lit/lit/display.py | 46 ++++++++++---
.../tests/Inputs/diff-test-update-retry/1.in | 1 +
.../Inputs/diff-test-update-retry/lit.cfg | 8 +++
.../multiple-split-file-enough-retries.in | 19 ++++++
.../multiple-split-file-enough-retries.out | 20 ++++++
.../multiple-split-file-no-enough-retries.out | 0
.../multiple-split-file-not-enough-retries.in | 20 ++++++
...multiple-split-file-not-enough-retries.out | 22 ++++++
.../multiple-split-file-unrelated-failure.in | 21 ++++++
.../multiple-split-file-unrelated-failure.out | 21 ++++++
.../single-split-file.in | 7 ++
.../single-split-file.out | 8 +++
.../utils/lit/tests/diff-test-update-retry.py | 67 +++++++++++++++++++
llvm/utils/lit/tests/diff-test-update.py | 12 ++--
17 files changed, 292 insertions(+), 32 deletions(-)
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/1.in
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.in
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-no-enough-retries.out
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.in
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.in
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/single-split-file.in
create mode 100644 llvm/utils/lit/tests/Inputs/diff-test-update-retry/single-split-file.out
create mode 100644 llvm/utils/lit/tests/diff-test-update-retry.py
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 83a09eb784fd3..e06c3610404b9 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -152,7 +152,13 @@ class Result(object):
"""Wrapper for the results of executing an individual test."""
def __init__(
- self, code, output="", elapsed=None, attempts=1, max_allowed_attempts=None
+ self,
+ code,
+ output="",
+ elapsed=None,
+ attempts=1,
+ max_allowed_attempts=None,
+ test_updater_outputs=[],
):
# The result code.
self.code = code
@@ -170,6 +176,8 @@ def __init__(
self.attempts = attempts
# How many attempts were allowed for this test
self.max_allowed_attempts = max_allowed_attempts
+ # Outputs from test updaters. One entry per attempt, or empty if disabled.
+ self.test_updater_outputs = test_updater_outputs
def addMetric(self, name, value):
"""
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 2683180d2864d..3e31e2eab07e1 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1190,8 +1190,9 @@ def formatOutput(title, data, limit=None):
# from the script, and there is no execution trace.
def executeScriptInternal(
test, litConfig, tmpBase, commands, cwd, debug=True
-) -> Tuple[str, str, int, Optional[str]]:
+) -> Tuple[str, str, int, Optional[str], Optional[str]]:
cmds = []
+ update_output = None
for i, ln in enumerate(commands):
# Within lit, we try to always add '%dbg(...)' to command lines in order
# to maximize debuggability. However, custom lit test formats might not
@@ -1326,14 +1327,14 @@ def executeScriptInternal(
output += traceback.format_exc()
raise TestUpdaterException(output)
if update_output:
- for line in update_output.splitlines():
- out += f"# {line}\n"
break
- return out, err, exitCode, timeoutInfo
+ return out, err, exitCode, timeoutInfo, update_output
-def executeScript(test, litConfig, tmpBase, commands, cwd):
+def executeScript(
+ test, litConfig, tmpBase, commands, cwd
+) -> Tuple[str, str, int, Optional[str], Optional[str]]:
bashPath = litConfig.getBashPath()
isWin32CMDEXE = litConfig.isWindows and not bashPath
script = tmpBase + ".script"
@@ -1429,9 +1430,9 @@ def executeScript(test, litConfig, tmpBase, commands, cwd):
env=env,
timeout=litConfig.maxIndividualTestTime,
)
- return (out, err, exitCode, None)
+ return (out, err, exitCode, None, None)
except lit.util.ExecuteCommandTimeoutException as e:
- return (e.out, e.err, e.exitCode, e.msg)
+ return (e.out, e.err, e.exitCode, e.msg, None)
def parseIntegratedTestScriptCommands(source_path, keywords):
@@ -2338,7 +2339,7 @@ def _runShTest(test, litConfig, useExternalSh, script, tmpBase) -> lit.Test.Resu
# Always returns the tuple (out, err, exitCode, timeoutInfo, status).
def runOnce(
execdir,
- ) -> Tuple[str, str, int, Optional[str], Test.ResultCode]:
+ ) -> Tuple[str, str, int, Optional[str], Test.ResultCode, Optional[str]]:
# script is modified below (for litConfig.per_test_coverage, and for
# %dbg expansions). runOnce can be called multiple times, but applying
# the modifications multiple times can corrupt script, so always modify
@@ -2374,9 +2375,9 @@ def runOnce(
)
except ScriptFatal as e:
out = f"# " + "\n# ".join(str(e).splitlines()) + "\n"
- return out, "", 1, None, Test.UNRESOLVED
+ return out, "", 1, None, Test.UNRESOLVED, None
- out, err, exitCode, timeoutInfo = res
+ out, err, exitCode, timeoutInfo, test_update_output = res
if exitCode == 0:
status = Test.PASS
else:
@@ -2384,7 +2385,7 @@ def runOnce(
status = Test.FAIL
else:
status = Test.TIMEOUT
- return out, err, exitCode, timeoutInfo, status
+ return out, err, exitCode, timeoutInfo, status, test_update_output
# Create the output directory if it does not already exist.
pathlib.Path(tmpBase).parent.mkdir(parents=True, exist_ok=True)
@@ -2392,9 +2393,11 @@ def runOnce(
# Re-run failed tests up to test.allowed_retries times.
execdir = os.path.dirname(test.getExecPath())
attempts = test.allowed_retries + 1
+ test_updates = []
for i in range(attempts):
res = runOnce(execdir)
- out, err, exitCode, timeoutInfo, status = res
+ out, err, exitCode, timeoutInfo, status, test_update_output = res
+ test_updates.append(test_update_output)
if status != Test.FAIL:
break
@@ -2417,7 +2420,11 @@ def runOnce(
output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
return lit.Test.Result(
- status, output, attempts=i + 1, max_allowed_attempts=attempts
+ status,
+ output,
+ attempts=i + 1,
+ max_allowed_attempts=attempts,
+ test_updater_outputs=test_updates,
)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index ee05847e2c765..2fe2ed60aca70 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -19,7 +19,8 @@ class TestOrder(enum.Enum):
class TestOutputLevel(enum.IntEnum):
OFF = 0
FAILED = 1
- ALL = 2
+ FAILED_OR_FLAKY = 2
+ ALL = 3
@classmethod
def create(cls, value):
@@ -27,6 +28,8 @@ def create(cls, value):
return cls.OFF
if value == "failed":
return cls.FAILED
+ if value == "failed-or-flaky":
+ return cls.FAILED_OR_FLAKY
if value == "all":
return cls.ALL
raise ValueError(f"invalid output level {repr(value)} of type {type(value)}")
@@ -112,7 +115,7 @@ def parse_args():
"--test-output",
help="Control whether the executed commands and their outputs are printed after each test has executed (default off). "
"If --print-result-after is set lower than the level given to --test-output, --print-result-after is raised to match.",
- choices=["off", "failed", "all"],
+ choices=["off", "failed", "failed-or-flaky", "all"],
default="off",
action=TestOutputAction,
)
@@ -120,7 +123,7 @@ def parse_args():
"--print-result-after",
help="Control which the executed test names and results are printed after each test has executed (default all). "
"If --test-output is set higher than the level given to --print-result-after, --test-output is lowered to match.",
- choices=["off", "failed", "all"],
+ choices=["off", "failed", "failed-or-flaky", "all"],
default="all",
action=TestOutputAction,
)
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 4dc04d93d3ea7..8aec448f8b2fe 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -1,5 +1,9 @@
import sys
+from argparse import Namespace
+from typing import Optional
+from lit.Test import Test
+
def create_display(opts, tests, total_tests, workers):
if opts.print_result_after == "off" and not opts.useProgressBar:
@@ -76,8 +80,24 @@ def clear(self, interrupted):
pass
+def shouldPrintInfo(infoOption, test):
+ if infoOption == "all":
+ return True
+ if test.isFailure():
+ return infoOption == "failed" or infoOption == "failed-or-flaky"
+ if test.result.attempts > 1:
+ return infoOption == "failed-or-flaky"
+ return False
+
+
class Display(object):
- def __init__(self, opts, tests, header, progress_bar):
+ def __init__(
+ self,
+ opts: Namespace,
+ tests: list[Test],
+ header: Optional[str],
+ progress_bar,
+ ):
self.opts = opts
self.num_tests = len(tests)
self.header = header
@@ -94,11 +114,7 @@ def print_header(self):
def update(self, test):
self.completed += 1
- show_result = (
- test.isFailure()
- and self.opts.print_result_after == "failed"
- or self.opts.print_result_after == "all"
- )
+ show_result = shouldPrintInfo(self.opts.print_result_after, test)
if show_result:
if self.progress_bar:
self.progress_bar.clear(interrupted=False)
@@ -133,10 +149,9 @@ def print_result(self, test):
)
)
+ print_result = shouldPrintInfo(self.opts.test_output, test)
# Show the test failure output, if requested.
- if (
- test.isFailure() and self.opts.test_output == "failed"
- ) or self.opts.test_output == "all":
+ if print_result:
if test.isFailure():
print("%s TEST '%s' FAILED %s" % ("*" * 20, test_name, "*" * 20))
out = test.result.output
@@ -157,6 +172,19 @@ def print_result(self, test):
print(out)
print("*" * 20)
+ # Report any automatic fixes of the test case
+ if test.result.test_updater_outputs:
+ had_any_updates = False
+ for i, updater_result in enumerate(test.result.test_updater_outputs):
+ if not updater_result:
+ continue
+ if test.result.attempts > 1:
+ print(f"[Attempt {i + 1}]")
+ print(updater_result)
+ had_any_updates = True
+ if had_any_updates:
+ print("*" * 10)
+
# Report test metrics, if present.
if test.result.metrics:
print("%s TEST '%s' RESULTS %s" % ("*" * 10, test_name, "*" * 10))
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/1.in b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/1.in
new file mode 100644
index 0000000000000..b7d6715e2df11
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/1.in
@@ -0,0 +1 @@
+FOO
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/lit.cfg b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/lit.cfg
new file mode 100644
index 0000000000000..4dd1f975de005
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/lit.cfg
@@ -0,0 +1,8 @@
+import lit.formats
+
+config.name = "diff-test-update-retry"
+config.suffixes = [".test"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.in b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.in
new file mode 100644
index 0000000000000..7425478111279
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.in
@@ -0,0 +1,19 @@
+# ALLOW_RETRIES: 5
+
+# RUN: split-file %s %t
+# RUN: cp %S/1.in %t/out.txt
+
+# RUN: diff %t/test2.expected %t/out.txt
+# RUN: diff %t/test3.expected %t/out.txt
+# RUN: diff %t/test4.expected %t/out.txt
+# RUN: diff %t/test5.expected %t/out.txt
+
+#--- test1.expected
+unrelated
+#--- test2.expected
+#--- test3.expected
+#--- test4.expected
+filler
+#--- test5.expected
+
+
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out
new file mode 100644
index 0000000000000..15868c5c977d6
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out
@@ -0,0 +1,20 @@
+# ALLOW_RETRIES: 5
+
+# RUN: split-file %s %t
+# RUN: cp %S/1.in %t/out.txt
+
+# RUN: diff %t/test2.expected %t/out.txt
+# RUN: diff %t/test3.expected %t/out.txt
+# RUN: diff %t/test4.expected %t/out.txt
+# RUN: diff %t/test5.expected %t/out.txt
+
+#--- test1.expected
+unrelated
+#--- test2.expected
+FOO
+#--- test3.expected
+FOO
+#--- test4.expected
+FOO
+#--- test5.expected
+FOO
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-no-enough-retries.out b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-no-enough-retries.out
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.in b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.in
new file mode 100644
index 0000000000000..f878a163a01a5
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.in
@@ -0,0 +1,20 @@
+# ALLOW_RETRIES: 2
+
+# RUN: split-file %s %t
+# RUN: cp %S/1.in %t/out.txt
+
+# RUN: diff %t/test2.expected %t/out.txt
+# RUN: diff %t/test3.expected %t/out.txt
+# RUN: diff %t/test4.expected %t/out.txt
+# RUN: diff %t/test5.expected %t/out.txt
+
+#--- test1.expected
+unrelated
+#--- test2.expected
+#--- test3.expected
+#--- test4.expected
+filler
+#--- test5.expected
+
+
+
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out
new file mode 100644
index 0000000000000..6e4a4f80ce4b0
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out
@@ -0,0 +1,22 @@
+# ALLOW_RETRIES: 2
+
+# RUN: split-file %s %t
+# RUN: cp %S/1.in %t/out.txt
+
+# RUN: diff %t/test2.expected %t/out.txt
+# RUN: diff %t/test3.expected %t/out.txt
+# RUN: diff %t/test4.expected %t/out.txt
+# RUN: diff %t/test5.expected %t/out.txt
+
+#--- test1.expected
+unrelated
+#--- test2.expected
+FOO
+#--- test3.expected
+FOO
+#--- test4.expected
+FOO
+#--- test5.expected
+
+
+
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.in b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.in
new file mode 100644
index 0000000000000..b544217e550c4
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.in
@@ -0,0 +1,21 @@
+# ALLOW_RETRIES: 4
+
+# RUN: split-file %s %t
+# RUN: cp %S/1.in %t/out.txt
+
+# RUN: diff %t/test2.expected %t/out.txt
+# RUN: diff %t/test3.expected %t/out.txt
+# RUN: diff %t/test4.expected %t/out.txt
+# RUN: diff %t/test5.expected %t/out.txt
+# RUN: not echo "fail with nothing left to updated"
+
+#--- test1.expected
+unrelated
+#--- test2.expected
+#--- test3.expected
+#--- test4.expected
+filler
+#--- test5.expected
+
+
+
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out
new file mode 100644
index 0000000000000..6fb026175dc4b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out
@@ -0,0 +1,21 @@
+# ALLOW_RETRIES: 4
+
+# RUN: split-file %s %t
+# RUN: cp %S/1.in %t/out.txt
+
+# RUN: diff %t/test2.expected %t/out.txt
+# RUN: diff %t/test3.expected %t/out.txt
+# RUN: diff %t/test4.expected %t/out.txt
+# RUN: diff %t/test5.expected %t/out.txt
+# RUN: not echo "fail with nothing left to updated"
+
+#--- test1.expected
+unrelated
+#--- test2.expected
+FOO
+#--- test3.expected
+FOO
+#--- test4.expected
+FOO
+#--- test5.expected
+FOO
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/single-split-file.in b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/single-split-file.in
new file mode 100644
index 0000000000000..861daecd173b0
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/single-split-file.in
@@ -0,0 +1,7 @@
+# ALLOW_RETRIES: 1
+
+# RUN: split-file %s %t
+# RUN: cp %S/1.in %t/out.txt
+# RUN: diff %t/test.expected %t/out.txt
+
+#--- test.expected
diff --git a/llvm/utils/lit/tests/Inputs/diff-test-update-retry/single-split-file.out b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/single-split-file.out
new file mode 100644
index 0000000000000..4492600e2ba11
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/diff-test-update-retry/single-split-file.out
@@ -0,0 +1,8 @@
+# ALLOW_RETRIES: 1
+
+# RUN: split-file %s %t
+# RUN: cp %S/1.in %t/out.txt
+# RUN: diff %t/test.expected %t/out.txt
+
+#--- test.expected
+FOO
diff --git a/llvm/utils/lit/tests/diff-test-update-retry.py b/llvm/utils/lit/tests/diff-test-update-retry.py
new file mode 100644
index 0000000000000..8385c9de205b1
--- /dev/null
+++ b/llvm/utils/lit/tests/diff-test-update-retry.py
@@ -0,0 +1,67 @@
+# RUN: rm -rf %t && mkdir -p %t
+
+# RUN: cp %S/Inputs/diff-test-update-retry/1.in %t/1.in
+# RUN: cp %S/Inputs/diff-test-update-retry/lit.cfg %t/lit.cfg
+#
+# RUN: cp %S/Inputs/diff-test-update-retry/single-split-file.in %t/single-split-file.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.in %t/multiple-split-file-enough-retries.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.in %t/multiple-split-file-not-enough-retries.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.in %t/multiple-split-file-unrelated-failure.test
+
+# RUN: not %{lit} --update-tests -v %t > %t/out.txt
+
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/single-split-file.out %t/single-split-file.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out %t/multiple-split-file-enough-retries.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out %t/multiple-split-file-not-enough-retries.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out %t/multiple-split-file-unrelated-failure.test
+
+# RUN: FileCheck %s < %t/out.txt
+
+# CHECK-LABEL: FLAKYPASS: diff-test-update-retry :: multiple-split-file-enough-retries.test
+# CHECK-SAME: (1 of 4, 5 of 6 attempts)
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 4]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: **********
+
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test
+# CHECK-SAME: (2 of 4, 3 of 3 attempts)
+# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-not-enough-retries.test' FAILED ********************
+# CHECK: ********************
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: **********
+
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test
+# CHECK-SAME: (3 of 4, 5 of 5 attempts)
+# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-unrelated-failure.test' FAILED ********************
+# CHECK: ********************
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 4]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: **********
+
+# CHECK-LABEL: FLAKYPASS: diff-test-update-retry :: single-split-file.test
+# CHECK-SAME: (4 of 4, 2 of 2 attempts)
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}single-split-file.test
+# CHECK-NEXT: **********
+
+# CHECK-NEXT: ********************
+# CHECK-NEXT: Failed Tests (2):
+# CHECK-NEXT: diff-test-update-retry :: multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: diff-test-update-retry :: multiple-split-file-unrelated-failure.test
diff --git a/llvm/utils/lit/tests/diff-test-update.py b/llvm/utils/lit/tests/diff-test-update.py
index e23d3879bb56c..836327d5da216 100644
--- a/llvm/utils/lit/tests/diff-test-update.py
+++ b/llvm/utils/lit/tests/diff-test-update.py
@@ -19,12 +19,12 @@
# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update/split-both.out %S/Inputs/diff-test-update/split-both.test
-# CHECK: # update-diff-test: could not deduce source and target from {{.*}}1.in and {{.*}}2.in
-# CHECK: # update-diff-test: could not deduce source and target from {{.*}}1.txt and {{.*}}2.txt
-# CHECK: # update-diff-test: copied {{.*}}my-file.txt to {{.*}}my-file.expected
-# CHECK: # update-diff-test: copied {{.*}}1.txt to {{.*}}empty.txt
-# CHECK: # update-diff-test: copied {{.*}}diff-tmp.test.tmp.txt to {{.*}}diff-t-out.txt
-# CHECK: # update-diff-test: copied {{.*}}unrelated-split.txt to {{.*}}unrelated-split.expected
+# CHECK: update-diff-test: could not deduce source and target from {{.*}}1.in and {{.*}}2.in
+# CHECK: update-diff-test: could not deduce source and target from {{.*}}1.txt and {{.*}}2.txt
+# CHECK: update-diff-test: copied {{.*}}my-file.txt to {{.*}}my-file.expected
+# CHECK: update-diff-test: copied {{.*}}1.txt to {{.*}}empty.txt
+# CHECK: update-diff-test: copied {{.*}}diff-tmp.test.tmp.txt to {{.*}}diff-t-out.txt
+# CHECK: update-diff-test: copied {{.*}}unrelated-split.txt to {{.*}}unrelated-split.expected
# CHECK: Failed: 14 (100.00%)
>From 517c67e70b07957aad01feafd9b70327d1164c05 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 11 Feb 2026 22:04:05 -0800
Subject: [PATCH 02/16] [utils] count updated tests that pass as FIXED instead
of FLAKYPASS
This is a purely semantic change: these two statuses behave identically,
but it is somewhat misleading to label a test that was failing and then
passed after being auto-fixed as "flaky".
---
llvm/utils/lit/lit/Test.py | 1 +
llvm/utils/lit/lit/TestRunner.py | 5 ++++-
llvm/utils/lit/lit/reports.py | 1 +
llvm/utils/lit/tests/diff-test-update-retry.py | 4 ++--
4 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index e06c3610404b9..23fc61756afcf 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -47,6 +47,7 @@ def __repr__(self):
UNSUPPORTED = ResultCode("UNSUPPORTED", "Unsupported", False)
PASS = ResultCode("PASS", "Passed", False)
FLAKYPASS = ResultCode("FLAKYPASS", "Passed With Retry", False)
+FIXED = ResultCode("FIXED", "Passed After Update", False)
XFAIL = ResultCode("XFAIL", "Expectedly Failed", False)
# Failures
UNRESOLVED = ResultCode("UNRESOLVED", "Unresolved", True)
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 3e31e2eab07e1..9e5c7d253a236 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -2404,7 +2404,10 @@ def runOnce(
# If we had to run the test more than once, count it as a flaky pass. These
# will be printed separately in the test summary.
if i > 0 and status == Test.PASS:
- status = Test.FLAKYPASS
+ if any(test_updates):
+ status = Test.FIXED
+ else:
+ status = Test.FLAKYPASS
# Form the output log.
output = f"Exit Code: {exitCode}\n"
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 6f8a782a40aa8..39398012d941d 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -213,6 +213,7 @@ def gen_resultdb_test_entry(
result_code == lit.Test.PASS
or result_code == lit.Test.XPASS
or result_code == lit.Test.FLAKYPASS
+ or result_code == lit.Test.FIXED
):
test_data["status"] = "PASS"
elif result_code == lit.Test.FAIL or result_code == lit.Test.XFAIL:
diff --git a/llvm/utils/lit/tests/diff-test-update-retry.py b/llvm/utils/lit/tests/diff-test-update-retry.py
index 8385c9de205b1..e02e01f1e8310 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry.py
@@ -17,7 +17,7 @@
# RUN: FileCheck %s < %t/out.txt
-# CHECK-LABEL: FLAKYPASS: diff-test-update-retry :: multiple-split-file-enough-retries.test
+# CHECK-LABEL: FIXED: diff-test-update-retry :: multiple-split-file-enough-retries.test
# CHECK-SAME: (1 of 4, 5 of 6 attempts)
# CHECK-NEXT: [Attempt 1]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
@@ -55,7 +55,7 @@
# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: **********
-# CHECK-LABEL: FLAKYPASS: diff-test-update-retry :: single-split-file.test
+# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test
# CHECK-SAME: (4 of 4, 2 of 2 attempts)
# CHECK-NEXT: [Attempt 1]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}single-split-file.test
>From 7a594391e31ac2d17e6b09bc7c8eaf401b65b96b Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 11 Feb 2026 22:25:28 -0800
Subject: [PATCH 03/16] [utils] print small divider between subsections, large
divider after
Previously we would print 20 asterisks as a divider before and after the
test output, and then print 10 asterisks after any subsections of extra
info belonging to that test. This gave the appearance of the subsections
belonging to the next test, since there would be a smaller divider
between the subsections and the next test than between the subsections
and the current test. This changes the logic such that subsections are
responsible for printing the divider before it, rather than the one
after it. This way we can ensure that the big divider always ends the
last subsection.
---
llvm/utils/lit/lit/display.py | 24 ++++++++++++-----
.../utils/lit/tests/diff-test-update-retry.py | 26 ++++++++-----------
2 files changed, 28 insertions(+), 22 deletions(-)
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 8aec448f8b2fe..92dccae280f88 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -149,6 +149,7 @@ def print_result(self, test):
)
)
+ has_printed_info = False
print_result = shouldPrintInfo(self.opts.test_output, test)
# Show the test failure output, if requested.
if print_result:
@@ -170,31 +171,38 @@ def print_result(self, test):
# in this case.
out = out.decode(encoding=sys.stdout.encoding, errors="ignore")
print(out)
- print("*" * 20)
+ has_printed_info = True
# Report any automatic fixes of the test case
- if test.result.test_updater_outputs:
- had_any_updates = False
+ if any(test.result.test_updater_outputs):
+ if has_printed_info:
+ print("*" * 10)
+ else:
+ has_printed_info = True
for i, updater_result in enumerate(test.result.test_updater_outputs):
if not updater_result:
continue
if test.result.attempts > 1:
print(f"[Attempt {i + 1}]")
print(updater_result)
- had_any_updates = True
- if had_any_updates:
- print("*" * 10)
# Report test metrics, if present.
if test.result.metrics:
+ if has_printed_info:
+ print("*" * 10)
+ else:
+ has_printed_info = True
print("%s TEST '%s' RESULTS %s" % ("*" * 10, test_name, "*" * 10))
items = sorted(test.result.metrics.items())
for metric_name, value in items:
print("%s: %s " % (metric_name, value.format()))
- print("*" * 10)
# Report micro-tests, if present
if test.result.microResults:
+ if has_printed_info:
+ print("*" * 10)
+ else:
+ has_printed_info = True
items = sorted(test.result.microResults.items())
for micro_test_name, micro_test in items:
print("%s MICRO-TEST: %s" % ("*" * 3, micro_test_name))
@@ -204,5 +212,7 @@ def print_result(self, test):
for metric_name, value in sorted_metrics:
print(" %s: %s " % (metric_name, value.format()))
+ if has_printed_info:
+ print("*" * 20)
# Ensure the output is flushed.
sys.stdout.flush()
diff --git a/llvm/utils/lit/tests/diff-test-update-retry.py b/llvm/utils/lit/tests/diff-test-update-retry.py
index e02e01f1e8310..ece7a987546ee 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry.py
@@ -15,10 +15,9 @@
# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out %t/multiple-split-file-not-enough-retries.test
# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out %t/multiple-split-file-unrelated-failure.test
-# RUN: FileCheck %s < %t/out.txt
+# RUN: FileCheck %s --match-full-lines < %t/out.txt
-# CHECK-LABEL: FIXED: diff-test-update-retry :: multiple-split-file-enough-retries.test
-# CHECK-SAME: (1 of 4, 5 of 6 attempts)
+# CHECK-LABEL: FIXED: diff-test-update-retry :: multiple-split-file-enough-retries.test (1 of 4, 5 of 6 attempts)
# CHECK-NEXT: [Attempt 1]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 2]
@@ -27,24 +26,22 @@
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 4]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
-# CHECK-NEXT: **********
+# CHECK-NEXT: ********************
-# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test
-# CHECK-SAME: (2 of 4, 3 of 3 attempts)
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test (2 of 4, 3 of 3 attempts)
# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-not-enough-retries.test' FAILED ********************
-# CHECK: ********************
+# CHECK: **********
# CHECK-NEXT: [Attempt 1]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 2]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 3]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
-# CHECK-NEXT: **********
+# CHECK-NEXT: ********************
-# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test
-# CHECK-SAME: (3 of 4, 5 of 5 attempts)
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-unrelated-failure.test' FAILED ********************
-# CHECK: ********************
+# CHECK: **********
# CHECK-NEXT: [Attempt 1]
# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
@@ -53,13 +50,12 @@
# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
-# CHECK-NEXT: **********
+# CHECK-NEXT: ********************
-# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test
-# CHECK-SAME: (4 of 4, 2 of 2 attempts)
+# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
# CHECK-NEXT: [Attempt 1]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}single-split-file.test
-# CHECK-NEXT: **********
+# CHECK-NEXT: ********************
# CHECK-NEXT: ********************
# CHECK-NEXT: Failed Tests (2):
>From 24709af21082ff3289a3c4afa4093e4982e98a15 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 11 Feb 2026 22:50:00 -0800
Subject: [PATCH 04/16] [utils] add test cases with varying verbosity (NFC)
---
...iff-test-update-retry-default-verbosity.py | 59 ++++++++++++++++
.../diff-test-update-retry-failed-or-flaky.py | 67 +++++++++++++++++++
.../lit/tests/diff-test-update-retry-quiet.py | 43 ++++++++++++
3 files changed, 169 insertions(+)
create mode 100644 llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
create mode 100644 llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
create mode 100644 llvm/utils/lit/tests/diff-test-update-retry-quiet.py
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py b/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
new file mode 100644
index 0000000000000..ee1eebcc226c1
--- /dev/null
+++ b/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
@@ -0,0 +1,59 @@
+# RUN: rm -rf %t && mkdir -p %t
+
+# RUN: cp %S/Inputs/diff-test-update-retry/1.in %t/1.in
+# RUN: cp %S/Inputs/diff-test-update-retry/lit.cfg %t/lit.cfg
+#
+# RUN: cp %S/Inputs/diff-test-update-retry/single-split-file.in %t/single-split-file.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.in %t/multiple-split-file-enough-retries.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.in %t/multiple-split-file-not-enough-retries.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.in %t/multiple-split-file-unrelated-failure.test
+
+# RUN: not %{lit} --update-tests %t > %t/out.txt
+
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/single-split-file.out %t/single-split-file.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out %t/multiple-split-file-enough-retries.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out %t/multiple-split-file-not-enough-retries.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out %t/multiple-split-file-unrelated-failure.test
+
+# RUN: FileCheck %s --match-full-lines < %t/out.txt
+
+# CHECK-LABEL: FIXED: diff-test-update-retry :: multiple-split-file-enough-retries.test (1 of 4, 5 of 6 attempts)
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 4]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: ********************
+
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test (2 of 4, 3 of 3 attempts)
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: ********************
+
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 4]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: ********************
+
+# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}single-split-file.test
+# CHECK-NEXT: ********************
+
+# CHECK-NEXT: ********************
+# CHECK-NEXT: Failed Tests (2):
+# CHECK-NEXT: diff-test-update-retry :: multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: diff-test-update-retry :: multiple-split-file-unrelated-failure.test
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py b/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
new file mode 100644
index 0000000000000..a95f0360ba18d
--- /dev/null
+++ b/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
@@ -0,0 +1,67 @@
+# RUN: rm -rf %t && mkdir -p %t
+
+# RUN: cp %S/Inputs/diff-test-update-retry/1.in %t/1.in
+# RUN: cp %S/Inputs/diff-test-update-retry/lit.cfg %t/lit.cfg
+#
+# RUN: cp %S/Inputs/diff-test-update-retry/single-split-file.in %t/single-split-file.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.in %t/multiple-split-file-enough-retries.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.in %t/multiple-split-file-not-enough-retries.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.in %t/multiple-split-file-unrelated-failure.test
+
+# RUN: not %{lit} --update-tests --test-output=failed-or-flaky %t > %t/out.txt
+
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/single-split-file.out %t/single-split-file.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out %t/multiple-split-file-enough-retries.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out %t/multiple-split-file-not-enough-retries.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out %t/multiple-split-file-unrelated-failure.test
+
+# RUN: FileCheck %s --match-full-lines < %t/out.txt
+
+# CHECK-LABEL: FIXED: diff-test-update-retry :: multiple-split-file-enough-retries.test (1 of 4, 5 of 6 attempts)
+# CHECK-NEXT: Exit Code: 0
+# CHECK: **********
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: [Attempt 4]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: ********************
+
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test (2 of 4, 3 of 3 attempts)
+# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-not-enough-retries.test' FAILED ********************
+# CHECK: **********
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: ********************
+
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
+# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-unrelated-failure.test' FAILED ********************
+# CHECK: **********
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 4]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: ********************
+
+# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
+# CHECK-NEXT: Exit Code: 0
+# CHECK: **********
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}single-split-file.test
+# CHECK-NEXT: ********************
+
+# CHECK-NEXT: ********************
+# CHECK-NEXT: Failed Tests (2):
+# CHECK-NEXT: diff-test-update-retry :: multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: diff-test-update-retry :: multiple-split-file-unrelated-failure.test
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-quiet.py b/llvm/utils/lit/tests/diff-test-update-retry-quiet.py
new file mode 100644
index 0000000000000..df295d94d1078
--- /dev/null
+++ b/llvm/utils/lit/tests/diff-test-update-retry-quiet.py
@@ -0,0 +1,43 @@
+# RUN: rm -rf %t && mkdir -p %t
+
+# RUN: cp %S/Inputs/diff-test-update-retry/1.in %t/1.in
+# RUN: cp %S/Inputs/diff-test-update-retry/lit.cfg %t/lit.cfg
+#
+# RUN: cp %S/Inputs/diff-test-update-retry/single-split-file.in %t/single-split-file.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.in %t/multiple-split-file-enough-retries.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.in %t/multiple-split-file-not-enough-retries.test
+# RUN: cp %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.in %t/multiple-split-file-unrelated-failure.test
+
+# RUN: not %{lit} --update-tests -q %t > %t/out.txt
+
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/single-split-file.out %t/single-split-file.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out %t/multiple-split-file-enough-retries.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out %t/multiple-split-file-not-enough-retries.test
+# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out %t/multiple-split-file-unrelated-failure.test
+
+# RUN: FileCheck %s --match-full-lines --implicit-check-not=update-diff-test --implicit-check-not=FIXED < %t/out.txt
+
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test (2 of 4, 3 of 3 attempts)
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: ********************
+
+# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
+# CHECK-NEXT: [Attempt 1]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 2]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 3]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: [Attempt 4]
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: ********************
+
+# CHECK-NEXT: ********************
+# CHECK-NEXT: Failed Tests (2):
+# CHECK-NEXT: diff-test-update-retry :: multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: diff-test-update-retry :: multiple-split-file-unrelated-failure.test
>From 293c69b9a33f8941394bad1c4b41729f04d69346 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 11 Feb 2026 23:10:20 -0800
Subject: [PATCH 05/16] [utils] specify slice name in DiffUpdater output
This makes it easier after the fact to figure out what happened,
especially when a test is updated multiple times.
---
llvm/utils/lit/lit/DiffUpdater.py | 7 +++---
...iff-test-update-retry-default-verbosity.py | 24 +++++++++----------
.../diff-test-update-retry-failed-or-flaky.py | 24 +++++++++----------
.../lit/tests/diff-test-update-retry-quiet.py | 14 +++++------
.../utils/lit/tests/diff-test-update-retry.py | 24 +++++++++----------
5 files changed, 47 insertions(+), 46 deletions(-)
diff --git a/llvm/utils/lit/lit/DiffUpdater.py b/llvm/utils/lit/lit/DiffUpdater.py
index 9e75e4b4513df..62dfa1c2f6fa1 100644
--- a/llvm/utils/lit/lit/DiffUpdater.py
+++ b/llvm/utils/lit/lit/DiffUpdater.py
@@ -34,10 +34,11 @@ def __str__(self):
class SplitFileTarget:
- def __init__(self, slice_start_idx, test_path, lines):
+ def __init__(self, slice_start_idx, test_path, lines, filename):
self.slice_start_idx = slice_start_idx
self.test_path = test_path
self.lines = lines
+ self.filename = filename
def copyFrom(self, source):
lines_before = self.lines[: self.slice_start_idx + 1]
@@ -58,7 +59,7 @@ def copyFrom(self, source):
f.write(l)
def __str__(self):
- return f"slice in {self.test_path}"
+ return f"slice {self.filename} in {self.test_path}"
@staticmethod
def get_target_dir(commands, test_path):
@@ -89,7 +90,7 @@ def create(path, commands, test_path, target_dir):
break
else:
return None
- return SplitFileTarget(idx, test_path, lines)
+ return SplitFileTarget(idx, test_path, lines, p)
@staticmethod
def _get_split_line_path(l):
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py b/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
index ee1eebcc226c1..997066c6ca807 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
@@ -19,38 +19,38 @@
# CHECK-LABEL: FIXED: diff-test-update-retry :: multiple-split-file-enough-retries.test (1 of 4, 5 of 6 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test5.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test (2 of 4, 3 of 3 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}single-split-file.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test.expected in {{.*}}single-split-file.test
# CHECK-NEXT: ********************
# CHECK-NEXT: ********************
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py b/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
index a95f0360ba18d..39f9f86711858 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
@@ -21,44 +21,44 @@
# CHECK-NEXT: Exit Code: 0
# CHECK: **********
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test5.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test (2 of 4, 3 of 3 attempts)
# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-not-enough-retries.test' FAILED ********************
# CHECK: **********
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-unrelated-failure.test' FAILED ********************
# CHECK: **********
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
# CHECK-NEXT: Exit Code: 0
# CHECK: **********
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}single-split-file.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test.expected in {{.*}}single-split-file.test
# CHECK-NEXT: ********************
# CHECK-NEXT: ********************
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-quiet.py b/llvm/utils/lit/tests/diff-test-update-retry-quiet.py
index df295d94d1078..acb42903f3ab7 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry-quiet.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry-quiet.py
@@ -19,22 +19,22 @@
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test (2 of 4, 3 of 3 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: ********************
# CHECK-NEXT: ********************
diff --git a/llvm/utils/lit/tests/diff-test-update-retry.py b/llvm/utils/lit/tests/diff-test-update-retry.py
index ece7a987546ee..bac3fb1c4c506 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry.py
@@ -19,42 +19,42 @@
# CHECK-LABEL: FIXED: diff-test-update-retry :: multiple-split-file-enough-retries.test (1 of 4, 5 of 6 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test5.expected in {{.*}}multiple-split-file-enough-retries.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-not-enough-retries.test (2 of 4, 3 of 3 attempts)
# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-not-enough-retries.test' FAILED ********************
# CHECK: **********
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}multiple-split-file-not-enough-retries.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-not-enough-retries.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-unrelated-failure.test' FAILED ********************
# CHECK: **********
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice in {{.*}}single-split-file.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test.expected in {{.*}}single-split-file.test
# CHECK-NEXT: ********************
# CHECK-NEXT: ********************
>From f7de502fd519766a440901f6c2d478e99fc02344 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 11 Feb 2026 23:13:40 -0800
Subject: [PATCH 06/16] remove path separator from test case (NFC)
---
.../lit/tests/diff-test-update-retry-default-verbosity.py | 8 ++++----
.../lit/tests/diff-test-update-retry-failed-or-flaky.py | 8 ++++----
llvm/utils/lit/tests/diff-test-update-retry-quiet.py | 8 ++++----
llvm/utils/lit/tests/diff-test-update-retry.py | 8 ++++----
4 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py b/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
index 997066c6ca807..e9929dfdb58c3 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry-default-verbosity.py
@@ -39,13 +39,13 @@
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py b/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
index 39f9f86711858..718e7b52bc024 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry-failed-or-flaky.py
@@ -45,13 +45,13 @@
# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-unrelated-failure.test' FAILED ********************
# CHECK: **********
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
diff --git a/llvm/utils/lit/tests/diff-test-update-retry-quiet.py b/llvm/utils/lit/tests/diff-test-update-retry-quiet.py
index acb42903f3ab7..21dc9a0d481ae 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry-quiet.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry-quiet.py
@@ -28,13 +28,13 @@
# CHECK-LABEL: FAIL: diff-test-update-retry :: multiple-split-file-unrelated-failure.test (3 of 4, 5 of 5 attempts)
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: ********************
# CHECK-NEXT: ********************
diff --git a/llvm/utils/lit/tests/diff-test-update-retry.py b/llvm/utils/lit/tests/diff-test-update-retry.py
index bac3fb1c4c506..15dace3470821 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry.py
@@ -43,13 +43,13 @@
# CHECK-NEXT: ******************** TEST 'diff-test-update-retry :: multiple-split-file-unrelated-failure.test' FAILED ********************
# CHECK: **********
# CHECK-NEXT: [Attempt 1]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 2]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test3.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 3]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test4.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: [Attempt 4]
-# CHECK-NEXT: update-diff-test: copied {{.*}}multiple-split-file-unrelated-failure.test.tmp/out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
+# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test5.expected in {{.*}}multiple-split-file-unrelated-failure.test
# CHECK-NEXT: ********************
# CHECK-LABEL: FIXED: diff-test-update-retry :: single-split-file.test (4 of 4, 2 of 2 attempts)
>From fc2aa00c2327d5855af013cd1de7fea940f96cde Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 11 Feb 2026 23:58:26 -0800
Subject: [PATCH 07/16] [utils] update libcxx lit
---
libcxx/utils/libcxx/test/format.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index 76e9115295b99..50ddb89a6bc99 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -36,7 +36,7 @@ def _checkBaseSubstitutions(substitutions):
def _executeScriptInternal(test, litConfig, commands):
"""
- Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands), or an appropriate lit.Test.Result
+ Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands, testUpdateOutput), or an appropriate lit.Test.Result
in case of an error while parsing the script.
TODO: This really should be easier to access from Lit itself
@@ -53,9 +53,9 @@ def _executeScriptInternal(test, litConfig, commands):
)
except lit.TestRunner.ScriptFatal as e:
res = ("", str(e), 127, None)
- (out, err, exitCode, timeoutInfo) = res
+ (out, err, exitCode, timeoutInfo, testUpdateOutput) = res
- return (out, err, exitCode, timeoutInfo, parsedCommands)
+ return (out, err, exitCode, timeoutInfo, parsedCommands, testUpdateOutput)
def _validateModuleDependencies(modules):
@@ -406,7 +406,7 @@ def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
yield generator
return
- (out, err, exitCode, _, _) = result
+ (out, err, exitCode, _, _, _) = result
if exitCode != 0:
raise RuntimeError(f"Error while trying to generate gen test {'/'.join(pathInSuite)}\nstdout:\n{out}\n\nstderr:\n{err}")
>From 6608f5229ee7329248a04c3fe54ac981d6c19301 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 00:03:13 -0800
Subject: [PATCH 08/16] DNM: debug windows CI failure
---
llvm/utils/lit/tests/diff-test-update-retry.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/utils/lit/tests/diff-test-update-retry.py b/llvm/utils/lit/tests/diff-test-update-retry.py
index 15dace3470821..1aae27076ecfd 100644
--- a/llvm/utils/lit/tests/diff-test-update-retry.py
+++ b/llvm/utils/lit/tests/diff-test-update-retry.py
@@ -10,13 +10,13 @@
# RUN: not %{lit} --update-tests -v %t > %t/out.txt
+# RUN: FileCheck %s --match-full-lines --dump-input-filter=all < %t/out.txt
+
# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/single-split-file.out %t/single-split-file.test
# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-enough-retries.out %t/multiple-split-file-enough-retries.test
# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-not-enough-retries.out %t/multiple-split-file-not-enough-retries.test
# RUN: diff --strip-trailing-cr %S/Inputs/diff-test-update-retry/multiple-split-file-unrelated-failure.out %t/multiple-split-file-unrelated-failure.test
-# RUN: FileCheck %s --match-full-lines < %t/out.txt
-
# CHECK-LABEL: FIXED: diff-test-update-retry :: multiple-split-file-enough-retries.test (1 of 4, 5 of 6 attempts)
# CHECK-NEXT: [Attempt 1]
# CHECK-NEXT: update-diff-test: copied {{.*}}out.txt to slice test2.expected in {{.*}}multiple-split-file-enough-retries.test
>From 0b261c70966d9ca7dfadfdf80a2e286549e3be10 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 16:32:17 -0800
Subject: [PATCH 09/16] update libcxx lit script
---
libcxx/utils/libcxx/test/dsl.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 88fc49160c56b..0751961c6e900 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -144,7 +144,7 @@ def sourceBuilds(config, source, additionalFlags=[]):
with _makeConfigTest(config) as test:
with open(test.getSourcePath(), "w") as sourceFile:
sourceFile.write(source)
- _, _, exitCode, _, _ = _executeWithFakeConfig(
+ _, _, exitCode, _, _, _ = _executeWithFakeConfig(
test, ["%{{build}} {}".format(" ".join(additionalFlags))]
)
return exitCode == 0
@@ -167,7 +167,7 @@ def programOutput(config, program, args=None):
with _makeConfigTest(config) as test:
with open(test.getSourcePath(), "w") as source:
source.write(program)
- _, err, exitCode, _, buildcmd = _executeWithFakeConfig(test, ["%{build}"])
+ _, err, exitCode, _, buildcmd, _ = _executeWithFakeConfig(test, ["%{build}"])
if exitCode != 0:
raise ConfigurationCompilationError(
"Failed to build program, cmd:\n{}\nstderr is:\n{}".format(
@@ -175,7 +175,7 @@ def programOutput(config, program, args=None):
)
)
- out, err, exitCode, _, runcmd = _executeWithFakeConfig(
+ out, err, exitCode, _, runcmd, _ = _executeWithFakeConfig(
test, ["%{{run}} {}".format(" ".join(args))]
)
if exitCode != 0:
@@ -212,7 +212,7 @@ def tryCompileFlag(config, flag):
"""
# fmt: off
with _makeConfigTest(config) as test:
- out, err, exitCode, timeoutInfo, _ = _executeWithFakeConfig(test, [
+ out, err, exitCode, timeoutInfo, _, _ = _executeWithFakeConfig(test, [
"%{{cxx}} -xc++ {} -Werror -fsyntax-only %{{flags}} %{{compile_flags}} {}".format(os.devnull, flag)
])
return exitCode, out, err
@@ -239,7 +239,7 @@ def runScriptExitCode(config, script):
could appear on the right-hand-side of a `RUN:` keyword.
"""
with _makeConfigTest(config) as test:
- _, _, exitCode, _, _ = _executeWithFakeConfig(test, script)
+ _, _, exitCode, _, _, _ = _executeWithFakeConfig(test, script)
return exitCode
@@ -253,7 +253,7 @@ def commandOutput(config, command):
could appear on the right-hand-side of a `RUN:` keyword.
"""
with _makeConfigTest(config) as test:
- out, err, exitCode, _, cmd = _executeWithFakeConfig(test, command)
+ out, err, exitCode, _, cmd, _ = _executeWithFakeConfig(test, command)
if exitCode != 0:
raise ConfigurationRuntimeError(
"Failed to run command: {}\nstderr is:\n{}".format(cmd, err)
@@ -330,7 +330,7 @@ def compilerMacros(config, flags=""):
#endif
"""
)
- unparsedOutput, err, exitCode, _, cmd = _executeWithFakeConfig(
+ unparsedOutput, err, exitCode, _, cmd, _ = _executeWithFakeConfig(
test, ["%{{cxx}} %s -dM -E %{{flags}} %{{compile_flags}} {}".format(flags)]
)
if exitCode != 0:
>From 1ae7214f3c7498a11a58f38746ab28e706658939 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 16:39:44 -0800
Subject: [PATCH 10/16] DNM: debug windows CI failure
---
llvm/utils/lit/lit/DiffUpdater.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/llvm/utils/lit/lit/DiffUpdater.py b/llvm/utils/lit/lit/DiffUpdater.py
index 62dfa1c2f6fa1..ae903e014af67 100644
--- a/llvm/utils/lit/lit/DiffUpdater.py
+++ b/llvm/utils/lit/lit/DiffUpdater.py
@@ -41,6 +41,8 @@ def __init__(self, slice_start_idx, test_path, lines, filename):
self.filename = filename
def copyFrom(self, source):
+ print(f"in copyFrom {source} -> {self.filename}")
+ print(f" file content before: {'\n'.join(self.lines)}")
lines_before = self.lines[: self.slice_start_idx + 1]
self.lines = self.lines[self.slice_start_idx + 1 :]
slice_end_idx = None
@@ -52,11 +54,15 @@ def copyFrom(self, source):
lines_after = self.lines[slice_end_idx:]
else:
lines_after = []
+ with open(source, "r") as f:
+ print(f" inserting file content: {f.read()}")
with open(source, "r") as f:
new_lines = lines_before + f.readlines() + lines_after
with open(self.test_path, "w") as f:
for l in new_lines:
+ print(f" writing out line: {l}")
f.write(l)
+ print("end copyFrom")
def __str__(self):
return f"slice {self.filename} in {self.test_path}"
>From 212c4504f44f9074247412b703f73c10daf85690 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 17:04:49 -0800
Subject: [PATCH 11/16] DNM: debug windows CI failure
---
llvm/utils/lit/lit/DiffUpdater.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/DiffUpdater.py b/llvm/utils/lit/lit/DiffUpdater.py
index ae903e014af67..d4070fc9d4ebf 100644
--- a/llvm/utils/lit/lit/DiffUpdater.py
+++ b/llvm/utils/lit/lit/DiffUpdater.py
@@ -42,7 +42,8 @@ def __init__(self, slice_start_idx, test_path, lines, filename):
def copyFrom(self, source):
print(f"in copyFrom {source} -> {self.filename}")
- print(f" file content before: {'\n'.join(self.lines)}")
+ tmp_lines = '\n'.join(self.lines)
+ print(f" file content before: {tmp_lines}")
lines_before = self.lines[: self.slice_start_idx + 1]
self.lines = self.lines[self.slice_start_idx + 1 :]
slice_end_idx = None
>From b742d7e24bcebb4910e83f2bb8b30346d1ff1065 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 18:41:01 -0800
Subject: [PATCH 12/16] DNM: debug windows CI failure
---
llvm/utils/lit/lit/DiffUpdater.py | 31 ++++++++++++++++++++++++++-----
1 file changed, 26 insertions(+), 5 deletions(-)
diff --git a/llvm/utils/lit/lit/DiffUpdater.py b/llvm/utils/lit/lit/DiffUpdater.py
index d4070fc9d4ebf..c09630b154082 100644
--- a/llvm/utils/lit/lit/DiffUpdater.py
+++ b/llvm/utils/lit/lit/DiffUpdater.py
@@ -35,26 +35,41 @@ def __str__(self):
class SplitFileTarget:
def __init__(self, slice_start_idx, test_path, lines, filename):
+ print(f"create SplitFile {filename} - i: {slice_start_idx}")
self.slice_start_idx = slice_start_idx
self.test_path = test_path
self.lines = lines
+ print(f" rest of lines start")
+ print("\n".join(self.lines[: self.slice_start_idx + 1]))
+ print(f" rest of lines end")
self.filename = filename
def copyFrom(self, source):
print(f"in copyFrom {source} -> {self.filename}")
- tmp_lines = '\n'.join(self.lines)
- print(f" file content before: {tmp_lines}")
+ print(f" file content start")
+ print("\n".join(self.lines))
+ print(f" file content end")
lines_before = self.lines[: self.slice_start_idx + 1]
self.lines = self.lines[self.slice_start_idx + 1 :]
+ print(f" before lines start")
+ print("\n".join(lines_before))
+ print(f" before lines end")
+ print(f" rest of lines start")
+ print("\n".join(self.lines))
+ print(f" rest of lines end")
slice_end_idx = None
for i, l in enumerate(self.lines):
if SplitFileTarget._get_split_line_path(l) != None:
+ print(f" end at idx {i}: {l}")
slice_end_idx = i
break
if slice_end_idx is not None:
lines_after = self.lines[slice_end_idx:]
else:
lines_after = []
+ print(f" lines after start")
+ print("\n".join(lines_after))
+ print(f" lines after end")
with open(source, "r") as f:
print(f" inserting file content: {f.read()}")
with open(source, "r") as f:
@@ -128,8 +143,12 @@ def get_source_and_target(a, b, test_path, commands):
a_target = None
b_target = None
if split_target_dir:
- a_target = SplitFileTarget.create(a, commands, test_path, split_target_dir)
- b_target = SplitFileTarget.create(b, commands, test_path, split_target_dir)
+ a_target = SplitFileTarget.create(
+ a, commands, test_path, split_target_dir
+ )
+ b_target = SplitFileTarget.create(
+ b, commands, test_path, split_target_dir
+ )
if a_target and not b_target:
return b, a_target
if b_target and not a_target:
@@ -166,7 +185,9 @@ def diff_test_updater(result, test, commands):
[cmd, a, b] = args
if cmd != "diff":
return None
- res = get_source_and_target(a, b, pathlib.Path(test.getFilePath()), commands)
+ res = get_source_and_target(
+ a, b, pathlib.Path(test.getFilePath()), commands
+ )
if not res:
return f"update-diff-test: could not deduce source and target from {a} and {b}"
source, target = res
>From 2b2efa22518606f27a99bd0e9668982012692d7c Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 19:37:45 -0800
Subject: [PATCH 13/16] DNM: debug windows CI failure
---
llvm/utils/lit/lit/DiffUpdater.py | 32 ++++++++++++++++++-------------
llvm/utils/lit/lit/display.py | 2 ++
2 files changed, 21 insertions(+), 13 deletions(-)
diff --git a/llvm/utils/lit/lit/DiffUpdater.py b/llvm/utils/lit/lit/DiffUpdater.py
index c09630b154082..9c38bc909bc1c 100644
--- a/llvm/utils/lit/lit/DiffUpdater.py
+++ b/llvm/utils/lit/lit/DiffUpdater.py
@@ -3,6 +3,8 @@
import shlex
import pathlib
+import time
+
"""
This file provides the `diff_test_updater` function, which is invoked on failed RUN lines when lit is executed with --update-tests.
It checks whether the failed command is `diff` and, if so, uses heuristics to determine which file is the checked-in reference file and which file is output from the test case.
@@ -39,46 +41,50 @@ def __init__(self, slice_start_idx, test_path, lines, filename):
self.slice_start_idx = slice_start_idx
self.test_path = test_path
self.lines = lines
- print(f" rest of lines start")
+ print(f"---- rest of lines start")
print("\n".join(self.lines[: self.slice_start_idx + 1]))
- print(f" rest of lines end")
+ print(f"---- rest of lines end")
self.filename = filename
def copyFrom(self, source):
print(f"in copyFrom {source} -> {self.filename}")
- print(f" file content start")
+ print(f"time start: {time.time()}")
+ print(f"---- file content start")
print("\n".join(self.lines))
- print(f" file content end")
+ print(f"---- file content end")
lines_before = self.lines[: self.slice_start_idx + 1]
self.lines = self.lines[self.slice_start_idx + 1 :]
- print(f" before lines start")
+ print(f"---- before lines start")
print("\n".join(lines_before))
- print(f" before lines end")
- print(f" rest of lines start")
+ print(f"---- before lines end")
+ print(f"---- rest of lines start")
print("\n".join(self.lines))
- print(f" rest of lines end")
+ print(f"---- rest of lines end")
slice_end_idx = None
+ print(f"---- checking line ")
for i, l in enumerate(self.lines):
+ print(f"-------- checking line: {l}")
if SplitFileTarget._get_split_line_path(l) != None:
- print(f" end at idx {i}: {l}")
+ print(f"------------ end at idx {i}: {l}")
slice_end_idx = i
break
if slice_end_idx is not None:
lines_after = self.lines[slice_end_idx:]
else:
lines_after = []
- print(f" lines after start")
+ print(f"---- lines after start")
print("\n".join(lines_after))
- print(f" lines after end")
+ print(f"---- lines after end")
with open(source, "r") as f:
- print(f" inserting file content: {f.read()}")
+ print(f"---- inserting file content: {f.read()}")
with open(source, "r") as f:
new_lines = lines_before + f.readlines() + lines_after
with open(self.test_path, "w") as f:
for l in new_lines:
- print(f" writing out line: {l}")
+ print(f"---- writing out line: {l}")
f.write(l)
print("end copyFrom")
+ print(f"time end: {time.time()}")
def __str__(self):
return f"slice {self.filename} in {self.test_path}"
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 92dccae280f88..24fc10002f71c 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -4,6 +4,7 @@
from typing import Optional
from lit.Test import Test
+import time
def create_display(opts, tests, total_tests, workers):
if opts.print_result_after == "off" and not opts.useProgressBar:
@@ -153,6 +154,7 @@ def print_result(self, test):
print_result = shouldPrintInfo(self.opts.test_output, test)
# Show the test failure output, if requested.
if print_result:
+ print(f"time: {time.time()}")
if test.isFailure():
print("%s TEST '%s' FAILED %s" % ("*" * 20, test_name, "*" * 20))
out = test.result.output
>From f14de277b9192481ad74907f143874c4e23f531b Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 20:31:41 -0800
Subject: [PATCH 14/16] DNM: debug windows CI failure
---
llvm/utils/lit/lit/DiffUpdater.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/DiffUpdater.py b/llvm/utils/lit/lit/DiffUpdater.py
index 9c38bc909bc1c..3d3720860df58 100644
--- a/llvm/utils/lit/lit/DiffUpdater.py
+++ b/llvm/utils/lit/lit/DiffUpdater.py
@@ -79,10 +79,19 @@ def copyFrom(self, source):
print(f"---- inserting file content: {f.read()}")
with open(source, "r") as f:
new_lines = lines_before + f.readlines() + lines_after
+ print(f"---- writing: {self.test_path}")
with open(self.test_path, "w") as f:
for l in new_lines:
- print(f"---- writing out line: {l}")
+ print(f"-------- writing out line: {l}")
f.write(l)
+ with open(self.test_path, "r") as f:
+ print(f"---- comparing lines")
+ written_lines = f.readlines()
+ for a, b in zip(new_lines, written_lines):
+ print(f"-------- a: {a}")
+ print(f"-------- b: {b}")
+ if a != b:
+ print("------------ diff!")
print("end copyFrom")
print(f"time end: {time.time()}")
>From ccd27096b29789b036d4419c0003e6799ae80daa Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 22:30:26 -0800
Subject: [PATCH 15/16] DNM: debug windows CI failure
---
llvm/utils/lit/lit/TestRunner.py | 30 +++++++++++++++++++++++-------
1 file changed, 23 insertions(+), 7 deletions(-)
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 9e5c7d253a236..f7fc1ec2bf955 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -197,7 +197,7 @@ def __init__(
self.outputFiles = list(outputFiles)
-def executeShCmd(cmd, shenv, results, timeout=0):
+def executeShCmd(cmd, shenv, results, timeout=0, prev_updated_file=None):
"""
Wrapper around _executeShCmd that handles
timeout
@@ -208,7 +208,7 @@ def executeShCmd(cmd, shenv, results, timeout=0):
if timeout > 0:
timeoutHelper.startTimer()
try:
- finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper)
+ finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper, prev_updated_file)
except InternalShellError:
e = sys.exc_info()[1]
finalExitCode = 127
@@ -749,7 +749,7 @@ def _replaceReadFile(match):
return arguments
-def _executeShCmd(cmd, shenv, results, timeoutHelper):
+def _executeShCmd(cmd, shenv, results, timeoutHelper, prev_updated_file=None):
if timeoutHelper.timeoutReached():
# Prevent further recursion if the timeout has been hit
# as we should try avoid launching more processes.
@@ -996,6 +996,11 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
old_umask = -1
if cmd_shenv.umask != -1:
old_umask = os.umask(cmd_shenv.umask)
+ if prev_updated_file:
+ with open(prev_updated_file, 'r') as f:
+ print(f"prev updated file before Popen: {prev_updated_file} start")
+ print(f.read())
+ print(f"prev updated file before Popen end\n")
procs.append(
subprocess.Popen(
args,
@@ -1189,7 +1194,7 @@ def formatOutput(title, data, limit=None):
# function), out contains only stdout from the script, err contains only stderr
# from the script, and there is no execution trace.
def executeScriptInternal(
- test, litConfig, tmpBase, commands, cwd, debug=True
+ test, litConfig, tmpBase, commands, cwd, debug=True, prev_updated_file=None
) -> Tuple[str, str, int, Optional[str], Optional[str]]:
cmds = []
update_output = None
@@ -1231,7 +1236,7 @@ def executeScriptInternal(
shenv.env["LIT_CURRENT_TESTCASE"] = test.getFullName()
exitCode, timeoutInfo = executeShCmd(
- cmd, shenv, results, timeout=litConfig.maxIndividualTestTime
+ cmd, shenv, results, timeout=litConfig.maxIndividualTestTime, prev_updated_file=prev_updated_file
)
out = err = ""
@@ -2337,9 +2342,11 @@ def parseIntegratedTestScript(test, additional_parsers=[], require_script=True):
def _runShTest(test, litConfig, useExternalSh, script, tmpBase) -> lit.Test.Result:
# Always returns the tuple (out, err, exitCode, timeoutInfo, status).
+ test_updates = []
def runOnce(
execdir,
) -> Tuple[str, str, int, Optional[str], Test.ResultCode, Optional[str]]:
+ print("--runOnce--\n")
# script is modified below (for litConfig.per_test_coverage, and for
# %dbg expansions). runOnce can be called multiple times, but applying
# the modifications multiple times can corrupt script, so always modify
@@ -2370,8 +2377,18 @@ def runOnce(
if useExternalSh:
res = executeScript(test, litConfig, tmpBase, scriptCopy, execdir)
else:
+ prev_updated_file = None
+ if test_updates:
+ prev_test_update = test_updates[-1]
+ if prev_test_update and "copied" in prev_test_update:
+ prev_updated_file = prev_test_update.split(" ")[-1]
+ if prev_updated_file:
+ with open(prev_updated_file, 'r') as f:
+ print(f"prev updated file: {prev_updated_file} start")
+ print(f.read())
+ print(f"prev updated file end\n")
res = executeScriptInternal(
- test, litConfig, tmpBase, scriptCopy, execdir
+ test, litConfig, tmpBase, scriptCopy, execdir, prev_updated_file=prev_updated_file
)
except ScriptFatal as e:
out = f"# " + "\n# ".join(str(e).splitlines()) + "\n"
@@ -2393,7 +2410,6 @@ def runOnce(
# Re-run failed tests up to test.allowed_retries times.
execdir = os.path.dirname(test.getExecPath())
attempts = test.allowed_retries + 1
- test_updates = []
for i in range(attempts):
res = runOnce(execdir)
out, err, exitCode, timeoutInfo, status, test_update_output = res
>From 56a8639c2bfe8d820cc2118138488fb756b6b83c Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 12 Feb 2026 23:13:44 -0800
Subject: [PATCH 16/16] DNM: debug windows CI failure
---
llvm/utils/lit/lit/TestRunner.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index f7fc1ec2bf955..ba595ce106ce8 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -209,7 +209,9 @@ def executeShCmd(cmd, shenv, results, timeout=0, prev_updated_file=None):
timeoutHelper.startTimer()
try:
finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper, prev_updated_file)
+ print(f"finalExitCode: {finalExitCode}")
except InternalShellError:
+ print("InternalShellError")
e = sys.exc_info()[1]
finalExitCode = 127
results.append(
@@ -1001,6 +1003,7 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper, prev_updated_file=None):
print(f"prev updated file before Popen: {prev_updated_file} start")
print(f.read())
print(f"prev updated file before Popen end\n")
+ print(f"will run Popen: {' '.join(args)}")
procs.append(
subprocess.Popen(
args,
@@ -1113,6 +1116,9 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper, prev_updated_file=None):
if data is not None:
output_files.append((name, path, data))
+ print(f"appending result (cmd): {cmd.commands[i]}")
+ print(f"appending result (out): {out}")
+ print(f"appending result (err): {err}")
results.append(
ShellCommandResult(
cmd.commands[i],
@@ -1240,7 +1246,9 @@ def executeScriptInternal(
)
out = err = ""
+ print("start enumerate(results)")
for i, result in enumerate(results):
+ print(f"\nin enumerate(results): {i}\n\n")
if not debug:
out += result.stdout
err += result.stderr
@@ -1333,6 +1341,7 @@ def executeScriptInternal(
raise TestUpdaterException(output)
if update_output:
break
+ print("end enumerate(results)")
return out, err, exitCode, timeoutInfo, update_output
More information about the llvm-commits
mailing list