[clang] [llvm] [Utils] Add new --update-tests flag to llvm-lit (PR #108425)
Henrik G. Olsson via cfe-commits
cfe-commits at lists.llvm.org
Fri Sep 13 20:10:37 PDT 2024
https://github.com/hnrklssn updated https://github.com/llvm/llvm-project/pull/108425
>From 451a178dbb461e6b3dd264be6ede0aa26283dbbe Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 28 Aug 2024 23:30:49 -0700
Subject: [PATCH 1/2] [Utils] Add --update-tests to lit
This adds a flag to lit for detecting and updating failing tests when
possible to do so automatically. The flag uses a plugin architecture
where config files can add additional auto-updaters for the types of
tests in the test suite. When a test fails with --update-tests enabled
lit passes the test RUN invocation and output to each registered test
updater until one of them signals that it updated the test. As such it
is the responsibility of the test updater to only update tests where it
is reasonably certain that it will actually fix the test, or come close
to doing so.
Also adds one initial test updater specific to clang's Sema test suite,
which updates the expected-[diag-kind] lines in tests using the -verify
flag.
---
clang/test/Sema/lit.local.cfg | 12 ++++++
.../Inputs/duplicate-diag.c | 2 +
.../Inputs/duplicate-diag.c.expected | 2 +
.../Inputs/infer-indentation.c | 2 +
.../Inputs/infer-indentation.c.expected | 2 +
.../Inputs/leave-existing-diags.c | 4 +-
.../Inputs/leave-existing-diags.c.expected | 4 +-
.../Inputs/multiple-errors.c | 2 +
.../Inputs/multiple-errors.c.expected | 2 +
.../multiple-missing-errors-same-line.c | 2 +
...ltiple-missing-errors-same-line.c.expected | 2 +
.../update-verify-tests/Inputs/no-checks.c | 2 +
.../Inputs/no-checks.c.expected | 2 +
.../update-verify-tests/Inputs/no-diags.c | 2 +
.../Inputs/no-diags.c.expected | 2 +
.../Inputs/no-expected-diags.c | 2 +
.../Inputs/no-expected-diags.c.expected | 2 +
.../Inputs/non-default-prefix.c | 2 +
.../Inputs/non-default-prefix.c.expected | 2 +
.../Inputs/update-same-line.c | 2 +
.../Inputs/update-same-line.c.expected | 2 +
.../Inputs/update-single-check.c | 2 +
.../Inputs/update-single-check.c.expected | 2 +
.../update-verify-tests/LitTests/.gitignore | 4 ++
.../update-verify-tests/LitTests/lit.cfg | 31 ++++++++++++++++
.../utils/update-verify-tests/lit-plugin.test | 5 +++
.../utils/update-verify-tests/lit.local.cfg | 11 ++++++
clang/utils/UpdateVerifyTests/__init__.py | 1 +
clang/utils/UpdateVerifyTests/litplugin.py | 37 +++++++++++++++++++
clang/utils/update-verify-tests.py | 3 ++
llvm/utils/lit/lit/LitConfig.py | 3 ++
llvm/utils/lit/lit/TestRunner.py | 12 ++++++
llvm/utils/lit/lit/cl_arguments.py | 6 +++
llvm/utils/lit/lit/llvm/config.py | 5 +++
llvm/utils/lit/lit/main.py | 1 +
35 files changed, 177 insertions(+), 2 deletions(-)
create mode 100644 clang/test/utils/update-verify-tests/LitTests/.gitignore
create mode 100644 clang/test/utils/update-verify-tests/LitTests/lit.cfg
create mode 100644 clang/test/utils/update-verify-tests/lit-plugin.test
create mode 100644 clang/utils/UpdateVerifyTests/__init__.py
create mode 100644 clang/utils/UpdateVerifyTests/litplugin.py
diff --git a/clang/test/Sema/lit.local.cfg b/clang/test/Sema/lit.local.cfg
index baf1b39ef238cd..b385fd92098a8e 100644
--- a/clang/test/Sema/lit.local.cfg
+++ b/clang/test/Sema/lit.local.cfg
@@ -2,3 +2,15 @@ config.substitutions = list(config.substitutions)
config.substitutions.insert(
0, (r"%clang\b", """*** Do not use the driver in Sema tests. ***""")
)
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ curdir = os.path.dirname(os.path.realpath(__file__))
+ testdir = os.path.dirname(curdir)
+ clangdir = os.path.dirname(testdir)
+ utilspath = os.path.join(clangdir, "utils")
+ sys.path.append(utilspath)
+ from UpdateVerifyTests.litplugin import verify_test_updater
+ lit_config.test_updaters.append(verify_test_updater)
diff --git a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
index 8c7e46c6eca9c1..d4a92eb4a7874a 100644
--- a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
+++ b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'a'}}
a = 2; a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
index 6214ff382f4495..5fd72709e487f5 100644
--- a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1 2{{use of undeclared identifier 'a'}}
a = 2; a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
index 0210ac35fd5cd1..9a58244d92e066 100644
--- a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
+++ b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1 2 {{use of undeclared identifier 'a'}}
a = 2; a = 2; b = 2; b = 2; c = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
index 5c5aaeeef97acf..127efb003b1883 100644
--- a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +3 {{use of undeclared identifier 'c'}}
// expected-error at +2 2 {{use of undeclared identifier 'b'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
index 1aa8d088e97273..075d6c80fd4c36 100644
--- a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
+++ b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
a = 2;
// expected-error at -1{{use of undeclared identifier 'a'}}
b = 2;// expected-error{{use of undeclared identifier 'b'}}
c = 2;
- // expected-error at 5{{use of undeclared identifier 'c'}}
+ // expected-error at 7{{use of undeclared identifier 'c'}}
d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
e = 2; // error to trigger mismatch
diff --git a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
index 6b621061bbfbbd..778ebd4ff4c7b9 100644
--- a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
a = 2;
// expected-error at -1{{use of undeclared identifier 'a'}}
b = 2;// expected-error{{use of undeclared identifier 'b'}}
c = 2;
- // expected-error at 5{{use of undeclared identifier 'c'}}
+ // expected-error at 7{{use of undeclared identifier 'c'}}
d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
// expected-error at +1{{use of undeclared identifier 'e'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
index e230e0a337bf49..d3fa93ec0e8221 100644
--- a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
a = 2;
b = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
index 27dc1f30a26faf..aa0cf22c8e8a7b 100644
--- a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'a'}}
a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
index 03f723d44bbe82..9c5104e693ae57 100644
--- a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
a = 2; b = 2; c = 2;
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
index 24b57f4353d95d..01076f76318997 100644
--- a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +3{{use of undeclared identifier 'c'}}
// expected-error at +2{{use of undeclared identifier 'b'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-checks.c b/clang/test/utils/update-verify-tests/Inputs/no-checks.c
index 8fd1f7cd333705..019698122c3f2b 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-checks.c
+++ b/clang/test/utils/update-verify-tests/Inputs/no-checks.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
bar = 2;
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
index e80548fbe50f2c..42c98d4f77b6ce 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'bar'}}
bar = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-diags.c b/clang/test/utils/update-verify-tests/Inputs/no-diags.c
index 66d169be439402..fddc2ad661fd2d 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-diags.c
+++ b/clang/test/utils/update-verify-tests/Inputs/no-diags.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{asdf}}
int a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
index 05230284945702..acb02badc5655a 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
@@ -1,4 +1,6 @@
// expected-no-diagnostics
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
int a = 2;
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
index 78b72e1357da76..e02b63dcd7a21b 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
+++ b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
// expected-no-diagnostics
void foo() {
a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
index d948ffce56189a..7fd85f2e064e28 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'a'}}
a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
index 3d63eaf0f1b878..b7a34e2dfa20d4 100644
--- a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
+++ b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify=check %s
+// RUN: diff %s %s.expected
void foo() {
a = 2; // check-error{{asdf}}
// expected-error at -1{ignored}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
index a877f86922123d..087a6be5ea4d2b 100644
--- a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify=check %s
+// RUN: diff %s %s.expected
void foo() {
a = 2; // check-error{{use of undeclared identifier 'a'}}
// expected-error at -1{ignored}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
index 5278ce0c57c319..e448800324e373 100644
--- a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
+++ b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
bar = 2; // expected-error {{asdf}}
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
index 8ba47f788319b1..ba4387a445fe08 100644
--- a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
bar = 2; // expected-error {{use of undeclared identifier 'bar'}}
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
index 20b011bfc3d77e..e093a8c2ddddbb 100644
--- a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
+++ b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{asdf}}
bar = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
index e80548fbe50f2c..42c98d4f77b6ce 100644
--- a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'bar'}}
bar = 2;
diff --git a/clang/test/utils/update-verify-tests/LitTests/.gitignore b/clang/test/utils/update-verify-tests/LitTests/.gitignore
new file mode 100644
index 00000000000000..307a6d636e7a28
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/LitTests/.gitignore
@@ -0,0 +1,4 @@
+*.c
+*.c.expected
+.lit_test_times.txt
+Output
diff --git a/clang/test/utils/update-verify-tests/LitTests/lit.cfg b/clang/test/utils/update-verify-tests/LitTests/lit.cfg
new file mode 100644
index 00000000000000..dbb22f13ef1303
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/LitTests/lit.cfg
@@ -0,0 +1,31 @@
+import lit.formats
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+from lit.llvm import llvm_config
+from os.path import dirname
+
+config.name = "Lit Update Verify Tests"
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.suffixes = [
+ ".c",
+]
+config.test_source_root = dirname(__file__)
+config.target_triple = None
+config.host_triple = None
+
+config.llvm_tools_dir = lit_config.path[0] # --path is explicitly passed by lit-plugin.test
+llvm_config.use_default_substitutions()
+llvm_config.use_clang()
+config.substitutions.append(("%clang_cc1", "%clang -cc1"))
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ config.clang_src_dir = dirname(dirname(dirname(dirname(config.test_source_root))))
+ utilspath = os.path.join(config.clang_src_dir, "utils")
+ sys.path.append(utilspath)
+ from UpdateVerifyTests.litplugin import verify_test_updater
+ # normally we'd append to the existing list, but when testing
+ # verify_test_updater we don't want diff_test_updater to accidentally interfere
+ lit_config.test_updaters = [verify_test_updater]
diff --git a/clang/test/utils/update-verify-tests/lit-plugin.test b/clang/test/utils/update-verify-tests/lit-plugin.test
new file mode 100644
index 00000000000000..f3bbf5ec36bfb5
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/lit-plugin.test
@@ -0,0 +1,5 @@
+# RUN: cp %S/Inputs/*.c %S/LitTests
+# RUN: cp %S/Inputs/*.c.expected %S/LitTests
+# RUN: not %{lit} %S/LitTests
+# RUN: not %{lit} %S/LitTests --update-tests
+# RUN: %{lit} %S/LitTests
diff --git a/clang/test/utils/update-verify-tests/lit.local.cfg b/clang/test/utils/update-verify-tests/lit.local.cfg
index a0b6afccc25010..281c5be37a983a 100644
--- a/clang/test/utils/update-verify-tests/lit.local.cfg
+++ b/clang/test/utils/update-verify-tests/lit.local.cfg
@@ -12,6 +12,7 @@ if config.standalone_build:
config.unsupported = True
else:
config.suffixes = [".test"]
+ config.excludes.add("LitTests")
script_path = os.path.join(
config.clang_src_dir, "utils", "update-verify-tests.py"
@@ -23,3 +24,13 @@ else:
"%s %s" % (python, shell_quote(script_path)),
)
)
+ config.substitutions.append(("%{lit}", "%{lit-no-order-opt} --order=lexical"))
+
+ lit_path = os.path.join(config.llvm_tools_dir, "llvm-lit")
+ lit_path = os.path.abspath(lit_path)
+ config.substitutions.append(
+ (
+ "%{lit-no-order-opt}",
+ f"{python} {lit_path} -j1 --path {config.llvm_tools_dir}"
+ )
+)
diff --git a/clang/utils/UpdateVerifyTests/__init__.py b/clang/utils/UpdateVerifyTests/__init__.py
new file mode 100644
index 00000000000000..8b137891791fe9
--- /dev/null
+++ b/clang/utils/UpdateVerifyTests/__init__.py
@@ -0,0 +1 @@
+
diff --git a/clang/utils/UpdateVerifyTests/litplugin.py b/clang/utils/UpdateVerifyTests/litplugin.py
new file mode 100644
index 00000000000000..31814acfa47e2a
--- /dev/null
+++ b/clang/utils/UpdateVerifyTests/litplugin.py
@@ -0,0 +1,37 @@
+import sys
+from lit.formats import ShTest
+from UpdateVerifyTests.core import check_expectations
+import re
+
+verify_r = re.compile(r"-verify(?:=(\w+))?")
+
+
+def get_verify_prefixes(command):
+ def get_default(prefix):
+ if prefix:
+ return prefix
+ return "expected"
+
+ prefixes = set()
+ for arg in command.args:
+ m = verify_r.match(arg)
+ if not m:
+ continue
+ prefix = m[1]
+ if not prefix:
+ prefix = "expected"
+ prefixes.add(prefix)
+ return prefixes
+
+
+def verify_test_updater(result):
+ if not result.stderr:
+ return None
+ prefixes = get_verify_prefixes(result.command)
+ if not prefixes:
+ return None
+ if len(prefixes) > 1:
+ return f"update-verify-test: not updating because of multiple prefixes - {prefixes}"
+ [prefix] = prefixes
+ return check_expectations(result.stderr.splitlines(), prefix)
+
diff --git a/clang/utils/update-verify-tests.py b/clang/utils/update-verify-tests.py
index e2874a8c049ef3..1507a284b01c5a 100644
--- a/clang/utils/update-verify-tests.py
+++ b/clang/utils/update-verify-tests.py
@@ -21,6 +21,9 @@
Example usage:
clang -verify [file] | python3 update-verify-tests.py
clang -verify=check [file] | python3 update-verify-tests.py --prefix check
+
+This can also be invoked automatically by lit for failing '-verify' tests in Sema by running:
+ llvm-lit --update-tests clang/test/Sema
"""
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index 5dc712ae28370c..198a2bf3172330 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -38,6 +38,7 @@ def __init__(
parallelism_groups={},
per_test_coverage=False,
gtest_sharding=True,
+ update_tests=False,
):
# The name of the test runner.
self.progname = progname
@@ -89,6 +90,8 @@ def __init__(
self.parallelism_groups = parallelism_groups
self.per_test_coverage = per_test_coverage
self.gtest_sharding = bool(gtest_sharding)
+ self.update_tests = update_tests
+ self.test_updaters = []
@property
def maxIndividualTestTime(self):
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index a1785073547ad0..7556433b83452c 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1190,6 +1190,18 @@ def executeScriptInternal(
str(result.timeoutReached),
)
+ if litConfig.update_tests:
+ for test_updater in litConfig.test_updaters:
+ try:
+ update_output = test_updater(result)
+ except Exception as e:
+ out += f"Exception occurred in test updater: {e}"
+ continue
+ if update_output:
+ for line in update_output.splitlines():
+ out += f"# {line}\n"
+ break
+
return out, err, exitCode, timeoutInfo
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index ed78256ee414b4..dcbe553c6d4827 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -204,6 +204,12 @@ def parse_args():
action="store_true",
help="Exit with status zero even if some tests fail",
)
+ execution_group.add_argument(
+ "--update-tests",
+ dest="update_tests",
+ action="store_true",
+ help="Try to update regression tests to reflect current behavior, if possible",
+ )
execution_test_time_group = execution_group.add_mutually_exclusive_group()
execution_test_time_group.add_argument(
"--skip-test-time-recording",
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index 5f762ec7f3514a..c05ec1664d4c45 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -64,12 +64,17 @@ def __init__(self, lit_config, config):
self.with_environment("_TAG_REDIR_ERR", "TXT")
self.with_environment("_CEE_RUNOPTS", "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)")
+ if lit_config.update_tests:
+ self.use_lit_shell = True
+
# Choose between lit's internal shell pipeline runner and a real shell.
# If LIT_USE_INTERNAL_SHELL is in the environment, we use that as an
# override.
lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL")
if lit_shell_env:
self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
+ if not self.use_lit_shell and lit_config.update_tests:
+ print("note: --update-tests is not supported when using external shell")
if not self.use_lit_shell:
features.add("shell")
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 24ba804f0c363f..745e376de7d529 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -42,6 +42,7 @@ def main(builtin_params={}):
config_prefix=opts.configPrefix,
per_test_coverage=opts.per_test_coverage,
gtest_sharding=opts.gtest_sharding,
+ update_tests=opts.update_tests,
)
discovered_tests = lit.discovery.find_tests_for_inputs(
>From b0cd104b2bafbd218fb67464ed97d2123ccf12bf Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 29 Aug 2024 17:11:45 -0700
Subject: [PATCH 2/2] [Utils] Add UTC support for lit's --update-tests
Adds support for invoking the appropriate update_*_test_checks.py script
from lit. Checks the header comment for which script was used to
generate it in the first place, so only test cases that were already
generated are affected.
To support this the interface for test updater functions is expanded to
not only take a ShellCommandResult, but also the Test object. This makes
it easy to get the file path of the current test.
Also adds a --path flag to update_any_test_checks.py as a convenience
to avoid having to manually set the PATH variable.
---
clang/test/lit.cfg.py | 10 ++++
clang/utils/UpdateVerifyTests/litplugin.py | 4 +-
llvm/test/lit.cfg.py | 10 ++++
llvm/utils/lit/lit/TestRunner.py | 2 +-
llvm/utils/update_any_test_checks.py | 54 ++++++++++++++++++++--
5 files changed, 74 insertions(+), 6 deletions(-)
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index 92a3361ce672e2..32ed5239b90795 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -362,3 +362,13 @@ def calculate_arch_features(arch_string):
# possibly be present in system and user configuration files, so disable
# default configs for the test runs.
config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1"
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ utilspath = os.path.join(config.llvm_src_root, "utils")
+ sys.path.append(utilspath)
+ from update_any_test_checks import utc_lit_plugin
+
+ lit_config.test_updaters.append(utc_lit_plugin)
diff --git a/clang/utils/UpdateVerifyTests/litplugin.py b/clang/utils/UpdateVerifyTests/litplugin.py
index 31814acfa47e2a..a01ab579b0165a 100644
--- a/clang/utils/UpdateVerifyTests/litplugin.py
+++ b/clang/utils/UpdateVerifyTests/litplugin.py
@@ -23,8 +23,8 @@ def get_default(prefix):
prefixes.add(prefix)
return prefixes
-
-def verify_test_updater(result):
+
+def verify_test_updater(result, test):
if not result.stderr:
return None
prefixes = get_verify_prefixes(result.command)
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 1e0dd0a7df34f1..ff11e4d1af5207 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -622,3 +622,13 @@ def have_ld64_plugin_support():
if config.has_logf128:
config.available_features.add("has_logf128")
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ utilspath = os.path.join(config.llvm_src_root, "utils")
+ sys.path.append(utilspath)
+ from update_any_test_checks import utc_lit_plugin
+
+ lit_config.test_updaters.append(utc_lit_plugin)
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 7556433b83452c..3a2cdc5026b0c2 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1193,7 +1193,7 @@ def executeScriptInternal(
if litConfig.update_tests:
for test_updater in litConfig.test_updaters:
try:
- update_output = test_updater(result)
+ update_output = test_updater(result, test)
except Exception as e:
out += f"Exception occurred in test updater: {e}"
continue
diff --git a/llvm/utils/update_any_test_checks.py b/llvm/utils/update_any_test_checks.py
index e8eef1a46c504f..76fe3365939290 100755
--- a/llvm/utils/update_any_test_checks.py
+++ b/llvm/utils/update_any_test_checks.py
@@ -34,9 +34,12 @@ def find_utc_tool(search_path, utc_name):
return None
-def run_utc_tool(utc_name, utc_tool, testname):
+def run_utc_tool(utc_name, utc_tool, testname, environment):
result = subprocess.run(
- [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+ [utc_tool, testname],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=environment,
)
return (result.returncode, result.stdout, result.stderr)
@@ -60,6 +63,42 @@ def expand_listfile_args(arg_list):
return exp_arg_list
+def utc_lit_plugin(result, test):
+ testname = test.getFilePath()
+ if not testname:
+ return None
+
+ script_name = os.path.abspath(__file__)
+ utc_search_path = os.path.join(os.path.dirname(script_name), os.path.pardir)
+
+ with open(testname, "r") as f:
+ header = f.readline().strip()
+
+ m = RE_ASSERTIONS.search(header)
+ if m is None:
+ return None
+
+ utc_name = m.group(1)
+ utc_tool = find_utc_tool([utc_search_path], utc_name)
+ if not utc_tool:
+ return f"update-utc-tests: {utc_name} not found"
+
+ return_code, stdout, stderr = run_utc_tool(
+ utc_name, utc_tool, testname, test.config.environment
+ )
+
+ stderr = stderr.decode(errors="replace")
+ if return_code != 0:
+ if stderr:
+ return f"update-utc-tests: {utc_name} exited with return code {return_code}\n{stderr.rstrip()}"
+ return f"update-utc-tests: {utc_name} exited with return code {return_code}"
+
+ stdout = stdout.decode(errors="replace")
+ if stdout:
+ return f"update-utc-tests: updated {testname}\n{stdout.rstrip()}"
+ return f"update-utc-tests: updated {testname}"
+
+
def main():
from argparse import RawTextHelpFormatter
@@ -78,6 +117,11 @@ def main():
nargs="*",
help="Additional directories to scan for update_*_test_checks scripts",
)
+ parser.add_argument(
+ "--path",
+ help="""Additional directories to scan for executables invoked by the update_*_test_checks scripts,
+separated by the platform path separator""",
+ )
parser.add_argument("tests", nargs="+")
config = parser.parse_args()
@@ -88,6 +132,10 @@ def main():
script_name = os.path.abspath(__file__)
utc_search_path.append(os.path.join(os.path.dirname(script_name), os.path.pardir))
+ local_env = os.environ.copy()
+ if config.path:
+ local_env["PATH"] = config.path + os.pathsep + local_env["PATH"]
+
not_autogenerated = []
utc_tools = {}
have_error = False
@@ -117,7 +165,7 @@ def main():
continue
future = executor.submit(
- run_utc_tool, utc_name, utc_tools[utc_name], testname
+ run_utc_tool, utc_name, utc_tools[utc_name], testname, local_env
)
jobs.append((testname, future))
More information about the cfe-commits
mailing list