[llvm] 6c3f18e - [Utils] Adds support for diff based tests to lit's --update-tests (#154147)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 8 15:46:51 PDT 2025
Author: Henrik G. Olsson
Date: 2025-09-08T15:46:48-07:00
New Revision: 6c3f18ebcaf234a842b8d169425ad73b93445d68
URL: https://github.com/llvm/llvm-project/commit/6c3f18ebcaf234a842b8d169425ad73b93445d68
DIFF: https://github.com/llvm/llvm-project/commit/6c3f18ebcaf234a842b8d169425ad73b93445d68.diff
LOG: [Utils] Adds support for diff based tests to lit's --update-tests (#154147)
This adds an updater to lit's --update-tests flag with support for
`diff`. If a RUN line containing the `diff` command fails, this function
will use heuristics to try to deduce which file is the "reference" file,
and copy the contents of the other file to the reference. If it cannot
deduce which file is the reference file, it does nothing.
The heuristics are currently:
- does one of the files end in .expected while the other does not? Then
the .expected file is the reference.
- does one of the file paths contain the substring ".tmp" while the
other does not? Then the file not containing ".tmp" is the reference.
This matches cases where one file path is constructed using the `%t`
substitution.
Added:
llvm/utils/lit/lit/DiffUpdater.py
llvm/utils/lit/tests/Inputs/diff-test-update/.gitignore
llvm/utils/lit/tests/Inputs/diff-test-update/1.in
llvm/utils/lit/tests/Inputs/diff-test-update/2.in
llvm/utils/lit/tests/Inputs/diff-test-update/diff-bail.test
llvm/utils/lit/tests/Inputs/diff-test-update/diff-bail2.test
llvm/utils/lit/tests/Inputs/diff-test-update/diff-expected.test
llvm/utils/lit/tests/Inputs/diff-test-update/diff-tmp-dir.test
llvm/utils/lit/tests/Inputs/diff-test-update/diff-tmp.test
llvm/utils/lit/tests/Inputs/diff-test-update/lit.cfg
llvm/utils/lit/tests/diff-test-update.py
Modified:
llvm/utils/lit/lit/LitConfig.py
Removed:
################################################################################
diff --git a/llvm/utils/lit/lit/DiffUpdater.py b/llvm/utils/lit/lit/DiffUpdater.py
new file mode 100644
index 0000000000000..de0001a94f0ba
--- /dev/null
+++ b/llvm/utils/lit/lit/DiffUpdater.py
@@ -0,0 +1,55 @@
+import shutil
+
+"""
+This file provides the `
diff _test_updater` function, which is invoked on failed RUN lines when lit is executed with --update-tests.
+It checks whether the failed command is `
diff ` and, if so, uses heuristics to determine which file is the checked-in reference file and which file is output from the test case.
+The heuristics are currently as follows:
+ - if exactly one file ends with ".expected" (common pattern in LLVM), that file is the reference file and the other is the output file
+ - if exactly one file path contains ".tmp" (e.g. because it contains the expansion of "%t"), that file is the reference file and the other is the output file
+If the command matches one of these patterns the output file content is copied to the reference file to make the test pass.
+Otherwise the test is ignored.
+
+Possible improvements:
+ - Support stdin patterns like "my_binary %s |
diff expected.txt"
+ - Scan RUN lines to see if a file is the source of output from a previous command.
+ If it is then it is not a reference file that can be copied to, regardless of name, since the test will overwrite it anyways.
+ - Only update the parts that need updating (based on the
diff output). Could help avoid noisy updates when e.g. whitespace changes are ignored.
+"""
+
+
+def get_source_and_target(a, b):
+ """
+ Try to figure out which file is the test output and which is the reference.
+ """
+ expected_suffix = ".expected"
+ if a.endswith(expected_suffix) and not b.endswith(expected_suffix):
+ return b, a
+ if b.endswith(expected_suffix) and not a.endswith(expected_suffix):
+ return a, b
+
+ tmp_substr = ".tmp"
+ if tmp_substr in a and not tmp_substr in b:
+ return a, b
+ if tmp_substr in b and not tmp_substr in a:
+ return b, a
+
+ return None
+
+
+def filter_flags(args):
+ return [arg for arg in args if not arg.startswith("-")]
+
+
+def
diff _test_updater(result, test):
+ args = filter_flags(result.command.args)
+ if len(args) != 3:
+ return None
+ [cmd, a, b] = args
+ if cmd != "
diff ":
+ return None
+ res = get_source_and_target(a, b)
+ if not res:
+ return f"update-
diff -test: could not deduce source and target from {a} and {b}"
+ source, target = res
+ shutil.copy(source, target)
+ return f"update-
diff -test: copied {source} to {target}"
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index df297b91be1b6..8cef3c1fd8569 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -8,6 +8,7 @@
import lit.formats
import lit.TestingConfig
import lit.util
+from lit.DiffUpdater import
diff _test_updater
# LitConfig must be a new style class for properties to work
class LitConfig(object):
@@ -93,7 +94,7 @@ def __init__(
self.per_test_coverage = per_test_coverage
self.gtest_sharding = bool(gtest_sharding)
self.update_tests = update_tests
- self.test_updaters = []
+ self.test_updaters = [
diff _test_updater]
@property
def maxIndividualTestTime(self):
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/.gitignore b/llvm/utils/lit/tests/Inputs/
diff -test-update/.gitignore
new file mode 100644
index 0000000000000..2211df63dd283
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/.gitignore
@@ -0,0 +1 @@
+*.txt
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/1.in b/llvm/utils/lit/tests/Inputs/
diff -test-update/1.in
new file mode 100644
index 0000000000000..b7d6715e2df11
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/1.in
@@ -0,0 +1 @@
+FOO
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/2.in b/llvm/utils/lit/tests/Inputs/
diff -test-update/2.in
new file mode 100644
index 0000000000000..ba578e48b1836
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/2.in
@@ -0,0 +1 @@
+BAR
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -bail.test b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -bail.test
new file mode 100644
index 0000000000000..ded931384f192
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -bail.test
@@ -0,0 +1,3 @@
+# There is no indication here of which file is the reference file to update
+# RUN:
diff %S/1.in %S/2.in
+
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -bail2.test b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -bail2.test
new file mode 100644
index 0000000000000..26e12a3b2b289
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -bail2.test
@@ -0,0 +1,7 @@
+# RUN: mkdir %t
+# RUN: cp %S/1.in %t/1.txt
+# RUN: cp %S/2.in %t/2.txt
+
+# There is no indication here of which file is the reference file to update
+# RUN:
diff %t/1.txt %t/2.txt
+
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -expected.test b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -expected.test
new file mode 100644
index 0000000000000..a26c6d338f964
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -expected.test
@@ -0,0 +1,5 @@
+# RUN: mkdir %t
+# RUN: cp %S/1.in %t/my-file.expected
+# RUN: cp %S/2.in %t/my-file.txt
+# RUN:
diff %t/my-file.expected %t/my-file.txt
+
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -tmp-dir.test b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -tmp-dir.test
new file mode 100644
index 0000000000000..929c2c1c6c7d3
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -tmp-dir.test
@@ -0,0 +1,6 @@
+# RUN: mkdir %t
+# RUN: touch %S/empty.txt
+# RUN: cp %S/1.in %t/1.txt
+
+# RUN:
diff %t/1.txt %S/empty.txt
+
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -tmp.test b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -tmp.test
new file mode 100644
index 0000000000000..042bf244ebaa1
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/
diff -tmp.test
@@ -0,0 +1,3 @@
+# RUN: cp %S/1.in %t.txt
+# RUN: cp %S/2.in %S/
diff -t-out.txt
+# RUN:
diff %t.txt %S/
diff -t-out.txt
diff --git a/llvm/utils/lit/tests/Inputs/
diff -test-update/lit.cfg b/llvm/utils/lit/tests/Inputs/
diff -test-update/lit.cfg
new file mode 100644
index 0000000000000..9bd255276638a
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/
diff -test-update/lit.cfg
@@ -0,0 +1,8 @@
+import lit.formats
+
+config.name = "
diff -test-update"
+config.suffixes = [".test"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
diff --git a/llvm/utils/lit/tests/
diff -test-update.py b/llvm/utils/lit/tests/
diff -test-update.py
new file mode 100644
index 0000000000000..21b869d120655
--- /dev/null
+++ b/llvm/utils/lit/tests/
diff -test-update.py
@@ -0,0 +1,10 @@
+# RUN: not %{lit} --update-tests -v %S/Inputs/
diff -test-update | FileCheck %s
+
+# CHECK: # update-
diff -test: could not deduce source and target from {{.*}}/Inputs/
diff -test-update/1.in and {{.*}}/Inputs/
diff -test-update/2.in
+# CHECK: # update-
diff -test: could not deduce source and target from {{.*}}/
diff -test-update/Output/
diff -bail2.test.tmp/1.txt and {{.*}}/
diff -test-update/Output/
diff -bail2.test.tmp/2.txt
+# CHECK: # update-
diff -test: copied {{.*}}/Output/
diff -expected.test.tmp/my-file.txt to {{.*}}/Output/
diff -expected.test.tmp/my-file.expected
+# CHECK: # update-
diff -test: copied {{.*}}/Output/
diff -tmp-dir.test.tmp/1.txt to {{.*}}/Inputs/
diff -test-update/empty.txt
+# CHECK: # update-
diff -test: copied {{.*}}/Inputs/
diff -test-update/Output/
diff -tmp.test.tmp.txt to {{.*}}/Inputs/
diff -test-update/
diff -t-out.txt
+
+
+# CHECK: Failed: 5 (100.00%)
More information about the llvm-commits
mailing list