[clang] [llvm] [Utils] Add new --update-tests flag to llvm-lit (PR #108425)
Henrik G. Olsson via cfe-commits
cfe-commits at lists.llvm.org
Thu Sep 12 10:07:07 PDT 2024
https://github.com/hnrklssn created https://github.com/llvm/llvm-project/pull/108425
This adds a flag to lit for detecting and updating failing tests when possible to do so automatically. The flag uses a plugin architecture where config files can add additional auto-updaters for the types of tests in the test suite. When a test fails with `--update-tests` enabled lit passes the test RUN invocation and output to each registered test updater until one of them signals that it updated the test (or all test updaters have been run). As such it is the responsibility of the test updater to only update tests where it is reasonably certain that it will actually fix the test, or come close to doing so.
Initially adds support for UpdateVerifyTests and UpdateTestChecks. The flag is currently only implemented for lit's internal shell, so `--update-tests` implies `LIT_USE_INTERNAL_SHELL=1`.
Builds on work in #97369
Fixes #81320
>From d93d77e193f235d12d4de4a4b184c458508fa8df Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Mon, 1 Jul 2024 18:19:09 -0700
Subject: [PATCH 1/6] [Utils] add update-verify-tests.py
Adds a python script to automatically take output from a failed clang
-verify test and update the test case(s) to expect the new behaviour.
---
clang/utils/update-verify-tests.py | 404 +++++++++++++++++++++++++++++
1 file changed, 404 insertions(+)
create mode 100644 clang/utils/update-verify-tests.py
diff --git a/clang/utils/update-verify-tests.py b/clang/utils/update-verify-tests.py
new file mode 100644
index 00000000000000..cfcfefc85e576a
--- /dev/null
+++ b/clang/utils/update-verify-tests.py
@@ -0,0 +1,404 @@
+import sys
+import re
+
+"""
+ Pipe output from clang's -verify into this script to have the test case updated to expect the actual diagnostic output.
+ When inserting new expected-* checks it will place them on the line before the location of the diagnostic, with an @+1,
+ or @+N for some N if there are multiple diagnostics emitted on the same line. If the current checks are using @-N for
+ this line, the new check will follow that convention also.
+ Existing checks will be left untouched as much as possible, including their location and whitespace content, to minimize
+ diffs. If inaccurate their count will be updated, or the check removed entirely.
+
+ Missing features:
+ - custom prefix support (-verify=my-prefix)
+ - multiple prefixes on the same line (-verify=my-prefix,my-other-prefix)
+ - multiple prefixes on separate RUN lines (RUN: -verify=my-prefix\nRUN: -verify my-other-prefix)
+ - regexes with expected-*-re: existing ones will be left untouched if accurate, but the script will abort if there are any
+ diagnostic mismatches on the same line.
+ - multiple checks targeting the same line are supported, but a line may only contain one check
+ - if multiple checks targeting the same line are failing the script is not guaranteed to produce a minimal diff
+
+Example usage:
+ build/bin/llvm-lit clang/test/Sema/ --no-progress-bar -v | python3 update-verify-tests.py
+"""
+
+
+class KnownException(Exception):
+ pass
+
+
+def parse_error_category(s):
+ parts = s.split("diagnostics")
+ diag_category = parts[0]
+ category_parts = parts[0].strip().strip("'").split("-")
+ expected = category_parts[0]
+ if expected != "expected":
+ raise Exception(
+ f"expected 'expected', but found '{expected}'. Custom verify prefixes are not supported."
+ )
+ diag_category = category_parts[1]
+ if "seen but not expected" in parts[1]:
+ seen = True
+ elif "expected but not seen" in parts[1]:
+ seen = False
+ else:
+ raise KnownException(f"unexpected category '{parts[1]}'")
+ return (diag_category, seen)
+
+
+diag_error_re = re.compile(r"File (\S+) Line (\d+): (.+)")
+diag_error_re2 = re.compile(r"File \S+ Line \d+ \(directive at (\S+):(\d+)\): (.+)")
+
+
+def parse_diag_error(s):
+ m = diag_error_re2.match(s)
+ if not m:
+ m = diag_error_re.match(s)
+ if not m:
+ return None
+ return (m.group(1), int(m.group(2)), m.group(3))
+
+
+class Line:
+ def __init__(self, content, line_n):
+ self.content = content
+ self.diag = None
+ self.line_n = line_n
+ self.related_diags = []
+ self.targeting_diags = []
+
+ def update_line_n(self, n):
+ if self.diag and not self.diag.line_is_absolute:
+ self.diag.orig_target_line_n += n - self.line_n
+ self.line_n = n
+ for diag in self.targeting_diags:
+ if diag.line_is_absolute:
+ diag.orig_target_line_n = n
+ else:
+ diag.orig_target_line_n = n - diag.line.line_n
+ for diag in self.related_diags:
+ if not diag.line_is_absolute:
+ pass
+
+ def render(self):
+ if not self.diag:
+ return self.content
+ assert "{{DIAG}}" in self.content
+ res = self.content.replace("{{DIAG}}", self.diag.render())
+ if not res.strip():
+ return ""
+ return res
+
+
+class Diag:
+ def __init__(
+ self,
+ diag_content,
+ category,
+ targeted_line_n,
+ line_is_absolute,
+ count,
+ line,
+ is_re,
+ whitespace_strings,
+ ):
+ self.diag_content = diag_content
+ self.category = category
+ self.orig_target_line_n = targeted_line_n
+ self.line_is_absolute = line_is_absolute
+ self.count = count
+ self.line = line
+ self.target = None
+ self.is_re = is_re
+ self.absolute_target()
+ self.whitespace_strings = whitespace_strings
+
+ def add(self):
+ if targeted_line > 0:
+ targeted_line += 1
+ elif targeted_line < 0:
+ targeted_line -= 1
+
+ def absolute_target(self):
+ if self.line_is_absolute:
+ res = self.orig_target_line_n
+ else:
+ res = self.line.line_n + self.orig_target_line_n
+ if self.target:
+ assert self.line.line_n == res
+ return res
+
+ def relative_target(self):
+ return self.absolute_target() - self.line.line_n
+
+ def render(self):
+ assert self.count >= 0
+ if self.count == 0:
+ return ""
+ line_location_s = ""
+ if self.relative_target() != 0:
+ if self.line_is_absolute:
+ line_location_s = f"@{self.absolute_target()}"
+ elif self.relative_target() > 0:
+ line_location_s = f"@+{self.relative_target()}"
+ else:
+ line_location_s = (
+ f"@{self.relative_target()}" # the minus sign is implicit
+ )
+ count_s = "" if self.count == 1 else f"{self.count}"
+ re_s = "-re" if self.is_re else ""
+ if self.whitespace_strings:
+ whitespace1_s = self.whitespace_strings[0]
+ whitespace2_s = self.whitespace_strings[1]
+ whitespace3_s = self.whitespace_strings[2]
+ whitespace4_s = self.whitespace_strings[3]
+ else:
+ whitespace1_s = " "
+ whitespace2_s = ""
+ whitespace3_s = ""
+ whitespace4_s = ""
+ if count_s and not whitespace3_s:
+ whitespace3_s = " "
+ return f"//{whitespace1_s}expected-{self.category}{re_s}{whitespace2_s}{line_location_s}{whitespace3_s}{count_s}{whitespace4_s}{{{{{self.diag_content}}}}}"
+
+
+expected_diag_re = re.compile(
+ r"//(\s*)expected-(note|warning|error)(-re)?(\s*)(@[+-]?\d+)?(\s*)(\d+)?(\s*)\{\{(.*)\}\}"
+)
+
+
+def parse_diag(line, filename, lines):
+ s = line.content
+ ms = expected_diag_re.findall(s)
+ if not ms:
+ return None
+ if len(ms) > 1:
+ print(
+ f"multiple diags on line {filename}:{line.line_n}. Aborting due to missing implementation."
+ )
+ sys.exit(1)
+ [
+ whitespace1_s,
+ category_s,
+ re_s,
+ whitespace2_s,
+ target_line_s,
+ whitespace3_s,
+ count_s,
+ whitespace4_s,
+ diag_s,
+ ] = ms[0]
+ if not target_line_s:
+ target_line_n = 0
+ is_absolute = False
+ elif target_line_s.startswith("@+"):
+ target_line_n = int(target_line_s[2:])
+ is_absolute = False
+ elif target_line_s.startswith("@-"):
+ target_line_n = int(target_line_s[1:])
+ is_absolute = False
+ else:
+ target_line_n = int(target_line_s[1:])
+ is_absolute = True
+ count = int(count_s) if count_s else 1
+ line.content = expected_diag_re.sub("{{DIAG}}", s)
+
+ return Diag(
+ diag_s,
+ category_s,
+ target_line_n,
+ is_absolute,
+ count,
+ line,
+ bool(re_s),
+ [whitespace1_s, whitespace2_s, whitespace3_s, whitespace4_s],
+ )
+
+
+def link_line_diags(lines, diag):
+ line_n = diag.line.line_n
+ target_line_n = diag.absolute_target()
+ step = 1 if target_line_n < line_n else -1
+ for i in range(target_line_n, line_n, step):
+ lines[i - 1].related_diags.append(diag)
+
+
+def add_line(new_line, lines):
+ lines.insert(new_line.line_n - 1, new_line)
+ for i in range(new_line.line_n, len(lines)):
+ line = lines[i]
+ assert line.line_n == i
+ line.update_line_n(i + 1)
+ assert all(line.line_n == i + 1 for i, line in enumerate(lines))
+
+
+indent_re = re.compile(r"\s*")
+
+
+def get_indent(s):
+ return indent_re.match(s).group(0)
+
+
+def add_diag(line_n, diag_s, diag_category, lines):
+ target = lines[line_n - 1]
+ for other in target.targeting_diags:
+ if other.is_re:
+ raise KnownException(
+ "mismatching diag on line with regex matcher. Skipping due to missing implementation"
+ )
+ reverse = (
+ True
+ if [other for other in target.targeting_diags if other.relative_target() < 0]
+ else False
+ )
+
+ targeting = [
+ other for other in target.targeting_diags if not other.line_is_absolute
+ ]
+ targeting.sort(reverse=reverse, key=lambda d: d.relative_target())
+ prev_offset = 0
+ prev_line = target
+ direction = -1 if reverse else 1
+ for d in targeting:
+ if d.relative_target() != prev_offset + direction:
+ break
+ prev_offset = d.relative_target()
+ prev_line = d.line
+ total_offset = prev_offset - 1 if reverse else prev_offset + 1
+ if reverse:
+ new_line_n = prev_line.line_n + 1
+ else:
+ new_line_n = prev_line.line_n
+ assert new_line_n == line_n + (not reverse) - total_offset
+
+ new_line = Line(get_indent(prev_line.content) + "{{DIAG}}\n", new_line_n)
+ new_line.related_diags = list(prev_line.related_diags)
+ add_line(new_line, lines)
+
+ new_diag = Diag(
+ diag_s, diag_category, total_offset, False, 1, new_line, False, None
+ )
+ new_line.diag = new_diag
+ new_diag.target_line = target
+ assert type(new_diag) != str
+ target.targeting_diags.append(new_diag)
+ link_line_diags(lines, new_diag)
+
+
+updated_test_files = set()
+
+
+def update_test_file(filename, diag_errors):
+ print(f"updating test file {filename}")
+ if filename in updated_test_files:
+ print(
+ f"{filename} already updated, but got new output - expect incorrect results"
+ )
+ else:
+ updated_test_files.add(filename)
+ with open(filename, "r") as f:
+ lines = [Line(line, i + 1) for i, line in enumerate(f.readlines())]
+ for line in lines:
+ diag = parse_diag(line, filename, lines)
+ if diag:
+ line.diag = diag
+ diag.target_line = lines[diag.absolute_target() - 1]
+ link_line_diags(lines, diag)
+ lines[diag.absolute_target() - 1].targeting_diags.append(diag)
+
+ for line_n, diag_s, diag_category, seen in diag_errors:
+ if seen:
+ continue
+ # this is a diagnostic expected but not seen
+ assert lines[line_n - 1].diag
+ if diag_s != lines[line_n - 1].diag.diag_content:
+ raise KnownException(
+ f"{filename}:{line_n} - found diag {lines[line_n - 1].diag.diag_content} but expected {diag_s}"
+ )
+ if diag_category != lines[line_n - 1].diag.category:
+ raise KnownException(
+ f"{filename}:{line_n} - found {lines[line_n - 1].diag.category} diag but expected {diag_category}"
+ )
+ lines[line_n - 1].diag.count -= 1
+ diag_errors_left = []
+ diag_errors.sort(reverse=True, key=lambda t: t[0])
+ for line_n, diag_s, diag_category, seen in diag_errors:
+ if not seen:
+ continue
+ target = lines[line_n - 1]
+ other_diags = [
+ d
+ for d in target.targeting_diags
+ if d.diag_content == diag_s and d.category == diag_category
+ ]
+ other_diag = other_diags[0] if other_diags else None
+ if other_diag:
+ other_diag.count += 1
+ else:
+ diag_errors_left.append((line_n, diag_s, diag_category))
+ for line_n, diag_s, diag_category in diag_errors_left:
+ add_diag(line_n, diag_s, diag_category, lines)
+ with open(filename, "w") as f:
+ for line in lines:
+ f.write(line.render())
+
+
+def update_test_files(errors):
+ errors_by_file = {}
+ for (filename, line, diag_s), (diag_category, seen) in errors:
+ if filename not in errors_by_file:
+ errors_by_file[filename] = []
+ errors_by_file[filename].append((line, diag_s, diag_category, seen))
+ for filename, diag_errors in errors_by_file.items():
+ try:
+ update_test_file(filename, diag_errors)
+ except KnownException as e:
+ print(f"{filename} - ERROR: {e}")
+ print("continuing...")
+
+
+curr = []
+curr_category = None
+curr_run_line = None
+lines_since_run = []
+for line in sys.stdin.readlines():
+ lines_since_run.append(line)
+ try:
+ if line.startswith("RUN:"):
+ if curr:
+ update_test_files(curr)
+ curr = []
+ lines_since_run = [line]
+ curr_run_line = line
+ else:
+ for line in lines_since_run:
+ print(line, end="")
+ print("====================")
+ print("no mismatching diagnostics found since last RUN line")
+ continue
+ if line.startswith("error: "):
+ if "no expected directives found" in line:
+ print(
+ f"no expected directives found for RUN line '{curr_run_line.strip()}'. Add 'expected-no-diagnostics' manually if this is intended."
+ )
+ continue
+ curr_category = parse_error_category(line[len("error: ") :])
+ continue
+
+ diag_error = parse_diag_error(line.strip())
+ if diag_error:
+ curr.append((diag_error, curr_category))
+ except Exception as e:
+ for line in lines_since_run:
+ print(line, end="")
+ print("====================")
+ print(e)
+ sys.exit(1)
+if curr:
+ update_test_files(curr)
+ print("done!")
+else:
+ for line in lines_since_run:
+ print(line, end="")
+ print("====================")
+ print("no mismatching diagnostics found")
>From e9919beb705213398ad4faaf4d2b70d6579bdf5a Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Tue, 27 Aug 2024 16:51:35 -0700
Subject: [PATCH 2/6] [Utils] Add testing for update-verify-tests, fix bugs
Fixes various line offset bugs. Fixes a bug where whitespace could be
emitted between 'expected-[category]' and the line location specifier,
which clang does not parse.
Now handles the cases where the file does not emit any diagnostics, or
previously contained 'expected-no-diagnostics'.
Also does further work to minimise diffs by replacing diagnostics checks
with a new one, even if the previous check was in a location that the
script would not normally emit. This is useful for the case where a
check is on the same line as the emitted diagnostic, and the diagnostic
message changes. Instead of removing the previous diagnostic and
inserting a new one with @+1, the old diagnostic text is replaced with
the new one.
---
.../Inputs/duplicate-diag.c | 8 +
.../Inputs/duplicate-diag.c.expected | 8 +
.../Inputs/infer-indentation.c | 8 +
.../Inputs/infer-indentation.c.expected | 11 +
.../Inputs/leave-existing-diags.c | 11 +
.../Inputs/leave-existing-diags.c.expected | 12 ++
.../Inputs/multiple-errors.c | 6 +
.../Inputs/multiple-errors.c.expected | 9 +
.../multiple-missing-errors-same-line.c | 8 +
...ltiple-missing-errors-same-line.c.expected | 13 ++
.../update-verify-tests/Inputs/no-checks.c | 3 +
.../Inputs/no-checks.c.expected | 4 +
.../update-verify-tests/Inputs/no-diags.c | 5 +
.../Inputs/no-diags.c.expected | 5 +
.../Inputs/no-expected-diags.c | 4 +
.../Inputs/no-expected-diags.c.expected | 4 +
.../Inputs/update-same-line.c | 4 +
.../Inputs/update-same-line.c.expected | 4 +
.../Inputs/update-single-check.c | 4 +
.../Inputs/update-single-check.c.expected | 4 +
.../update-verify-tests/duplicate-diag.test | 4 +
.../infer-indentation.test | 3 +
.../leave-existing-diags.test | 4 +
.../utils/update-verify-tests/lit.local.cfg | 25 +++
.../update-verify-tests/multiple-errors.test | 3 +
.../multiple-missing-errors-same-line.test | 3 +
.../utils/update-verify-tests/no-checks.test | 3 +
.../utils/update-verify-tests/no-diags.test | 4 +
.../no-expected-diags.test | 4 +
.../update-verify-tests/update-same-line.test | 4 +
.../update-single-check.test | 3 +
clang/utils/update-verify-tests.py | 197 ++++++++++++------
32 files changed, 325 insertions(+), 67 deletions(-)
create mode 100644 clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/no-checks.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/no-diags.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/update-same-line.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
create mode 100644 clang/test/utils/update-verify-tests/Inputs/update-single-check.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
create mode 100644 clang/test/utils/update-verify-tests/duplicate-diag.test
create mode 100644 clang/test/utils/update-verify-tests/infer-indentation.test
create mode 100644 clang/test/utils/update-verify-tests/leave-existing-diags.test
create mode 100644 clang/test/utils/update-verify-tests/lit.local.cfg
create mode 100644 clang/test/utils/update-verify-tests/multiple-errors.test
create mode 100644 clang/test/utils/update-verify-tests/multiple-missing-errors-same-line.test
create mode 100644 clang/test/utils/update-verify-tests/no-checks.test
create mode 100644 clang/test/utils/update-verify-tests/no-diags.test
create mode 100644 clang/test/utils/update-verify-tests/no-expected-diags.test
create mode 100644 clang/test/utils/update-verify-tests/update-same-line.test
create mode 100644 clang/test/utils/update-verify-tests/update-single-check.test
diff --git a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
new file mode 100644
index 00000000000000..8c7e46c6eca9c1
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
@@ -0,0 +1,8 @@
+void foo() {
+ // expected-error at +1{{use of undeclared identifier 'a'}}
+ a = 2; a = 2;
+ b = 2; b = 2;
+ // expected-error at +1 3{{use of undeclared identifier 'c'}}
+ c = 2; c = 2;
+ // expected-error 2{{asdf}}
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
new file mode 100644
index 00000000000000..6214ff382f4495
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
@@ -0,0 +1,8 @@
+void foo() {
+ // expected-error at +1 2{{use of undeclared identifier 'a'}}
+ a = 2; a = 2;
+ // expected-error at +1 2{{use of undeclared identifier 'b'}}
+ b = 2; b = 2;
+ // expected-error at +1 2{{use of undeclared identifier 'c'}}
+ c = 2; c = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
new file mode 100644
index 00000000000000..0210ac35fd5cd1
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
@@ -0,0 +1,8 @@
+void foo() {
+ // expected-error at +1 2 {{use of undeclared identifier 'a'}}
+ a = 2; a = 2; b = 2; b = 2; c = 2;
+ // expected-error at +1 2 {{asdf}}
+ d = 2;
+ e = 2; f = 2; // expected-error 2 {{use of undeclared identifier 'e'}}
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
new file mode 100644
index 00000000000000..5c5aaeeef97acf
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
@@ -0,0 +1,11 @@
+void foo() {
+ // expected-error at +3 {{use of undeclared identifier 'c'}}
+ // expected-error at +2 2 {{use of undeclared identifier 'b'}}
+ // expected-error at +1 2 {{use of undeclared identifier 'a'}}
+ a = 2; a = 2; b = 2; b = 2; c = 2;
+ // expected-error at +1 {{use of undeclared identifier 'd'}}
+ d = 2;
+ // expected-error at +1 {{use of undeclared identifier 'f'}}
+ e = 2; f = 2; // expected-error {{use of undeclared identifier 'e'}}
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
new file mode 100644
index 00000000000000..1aa8d088e97273
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
@@ -0,0 +1,11 @@
+void foo() {
+ a = 2;
+ // expected-error at -1{{use of undeclared identifier 'a'}}
+ b = 2;// expected-error{{use of undeclared identifier 'b'}}
+ c = 2;
+ // expected-error at 5{{use of undeclared identifier 'c'}}
+ d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
+
+ e = 2; // error to trigger mismatch
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
new file mode 100644
index 00000000000000..6b621061bbfbbd
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
@@ -0,0 +1,12 @@
+void foo() {
+ a = 2;
+ // expected-error at -1{{use of undeclared identifier 'a'}}
+ b = 2;// expected-error{{use of undeclared identifier 'b'}}
+ c = 2;
+ // expected-error at 5{{use of undeclared identifier 'c'}}
+ d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
+
+ // expected-error at +1{{use of undeclared identifier 'e'}}
+ e = 2; // error to trigger mismatch
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
new file mode 100644
index 00000000000000..e230e0a337bf49
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
@@ -0,0 +1,6 @@
+void foo() {
+ a = 2;
+ b = 2;
+
+ c = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
new file mode 100644
index 00000000000000..27dc1f30a26faf
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
@@ -0,0 +1,9 @@
+void foo() {
+ // expected-error at +1{{use of undeclared identifier 'a'}}
+ a = 2;
+ // expected-error at +1{{use of undeclared identifier 'b'}}
+ b = 2;
+
+ // expected-error at +1{{use of undeclared identifier 'c'}}
+ c = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
new file mode 100644
index 00000000000000..03f723d44bbe82
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
@@ -0,0 +1,8 @@
+void foo() {
+ a = 2; b = 2; c = 2;
+}
+
+void bar() {
+ x = 2; y = 2; z = 2;
+ // expected-error at -1{{use of undeclared identifier 'x'}}
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
new file mode 100644
index 00000000000000..24b57f4353d95d
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
@@ -0,0 +1,13 @@
+void foo() {
+ // expected-error at +3{{use of undeclared identifier 'c'}}
+ // expected-error at +2{{use of undeclared identifier 'b'}}
+ // expected-error at +1{{use of undeclared identifier 'a'}}
+ a = 2; b = 2; c = 2;
+}
+
+void bar() {
+ x = 2; y = 2; z = 2;
+ // expected-error at -1{{use of undeclared identifier 'x'}}
+ // expected-error at -2{{use of undeclared identifier 'y'}}
+ // expected-error at -3{{use of undeclared identifier 'z'}}
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-checks.c b/clang/test/utils/update-verify-tests/Inputs/no-checks.c
new file mode 100644
index 00000000000000..8fd1f7cd333705
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/no-checks.c
@@ -0,0 +1,3 @@
+void foo() {
+ bar = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
new file mode 100644
index 00000000000000..e80548fbe50f2c
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
@@ -0,0 +1,4 @@
+void foo() {
+ // expected-error at +1{{use of undeclared identifier 'bar'}}
+ bar = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-diags.c b/clang/test/utils/update-verify-tests/Inputs/no-diags.c
new file mode 100644
index 00000000000000..66d169be439402
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/no-diags.c
@@ -0,0 +1,5 @@
+void foo() {
+ // expected-error at +1{{asdf}}
+ int a = 2;
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
new file mode 100644
index 00000000000000..05230284945702
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
@@ -0,0 +1,5 @@
+// expected-no-diagnostics
+void foo() {
+ int a = 2;
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
new file mode 100644
index 00000000000000..78b72e1357da76
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
@@ -0,0 +1,4 @@
+// expected-no-diagnostics
+void foo() {
+ a = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
new file mode 100644
index 00000000000000..d948ffce56189a
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
@@ -0,0 +1,4 @@
+void foo() {
+ // expected-error at +1{{use of undeclared identifier 'a'}}
+ a = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
new file mode 100644
index 00000000000000..5278ce0c57c319
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
@@ -0,0 +1,4 @@
+void foo() {
+ bar = 2; // expected-error {{asdf}}
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
new file mode 100644
index 00000000000000..8ba47f788319b1
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
@@ -0,0 +1,4 @@
+void foo() {
+ bar = 2; // expected-error {{use of undeclared identifier 'bar'}}
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
new file mode 100644
index 00000000000000..20b011bfc3d77e
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
@@ -0,0 +1,4 @@
+void foo() {
+ // expected-error at +1{{asdf}}
+ bar = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
new file mode 100644
index 00000000000000..e80548fbe50f2c
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
@@ -0,0 +1,4 @@
+void foo() {
+ // expected-error at +1{{use of undeclared identifier 'bar'}}
+ bar = 2;
+}
diff --git a/clang/test/utils/update-verify-tests/duplicate-diag.test b/clang/test/utils/update-verify-tests/duplicate-diag.test
new file mode 100644
index 00000000000000..3163ce46199c3f
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/duplicate-diag.test
@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/duplicate-diag.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/duplicate-diag.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+
diff --git a/clang/test/utils/update-verify-tests/infer-indentation.test b/clang/test/utils/update-verify-tests/infer-indentation.test
new file mode 100644
index 00000000000000..6ba2f5d9d505bf
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/infer-indentation.test
@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/infer-indentation.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/infer-indentation.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
diff --git a/clang/test/utils/update-verify-tests/leave-existing-diags.test b/clang/test/utils/update-verify-tests/leave-existing-diags.test
new file mode 100644
index 00000000000000..cde690ef715a67
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/leave-existing-diags.test
@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/leave-existing-diags.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/leave-existing-diags.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+
diff --git a/clang/test/utils/update-verify-tests/lit.local.cfg b/clang/test/utils/update-verify-tests/lit.local.cfg
new file mode 100644
index 00000000000000..a0b6afccc25010
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/lit.local.cfg
@@ -0,0 +1,25 @@
+import lit.util
+
+# python 2.7 backwards compatibility
+try:
+ from shlex import quote as shell_quote
+except ImportError:
+ from pipes import quote as shell_quote
+
+if config.standalone_build:
+ # These tests require the update-verify-tests.py script from the clang
+ # source tree, so skip these tests if we are doing standalone builds.
+ config.unsupported = True
+else:
+ config.suffixes = [".test"]
+
+ script_path = os.path.join(
+ config.clang_src_dir, "utils", "update-verify-tests.py"
+ )
+ python = shell_quote(config.python_executable)
+ config.substitutions.append(
+ (
+ "%update-verify-tests",
+ "%s %s" % (python, shell_quote(script_path)),
+ )
+ )
diff --git a/clang/test/utils/update-verify-tests/multiple-errors.test b/clang/test/utils/update-verify-tests/multiple-errors.test
new file mode 100644
index 00000000000000..1332ef365dc863
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/multiple-errors.test
@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/multiple-errors.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/multiple-errors.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
diff --git a/clang/test/utils/update-verify-tests/multiple-missing-errors-same-line.test b/clang/test/utils/update-verify-tests/multiple-missing-errors-same-line.test
new file mode 100644
index 00000000000000..a9c21cd77e192b
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/multiple-missing-errors-same-line.test
@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/multiple-missing-errors-same-line.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/multiple-missing-errors-same-line.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
diff --git a/clang/test/utils/update-verify-tests/no-checks.test b/clang/test/utils/update-verify-tests/no-checks.test
new file mode 100644
index 00000000000000..f6ea91fa552be4
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/no-checks.test
@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/no-checks.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/no-checks.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
diff --git a/clang/test/utils/update-verify-tests/no-diags.test b/clang/test/utils/update-verify-tests/no-diags.test
new file mode 100644
index 00000000000000..464fe8894253b6
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/no-diags.test
@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/no-diags.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/no-diags.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+
diff --git a/clang/test/utils/update-verify-tests/no-expected-diags.test b/clang/test/utils/update-verify-tests/no-expected-diags.test
new file mode 100644
index 00000000000000..75235f17a64a29
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/no-expected-diags.test
@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/no-expected-diags.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/no-expected-diags.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+
diff --git a/clang/test/utils/update-verify-tests/update-same-line.test b/clang/test/utils/update-verify-tests/update-same-line.test
new file mode 100644
index 00000000000000..324768eae5faac
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/update-same-line.test
@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/update-same-line.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/update-same-line.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+
diff --git a/clang/test/utils/update-verify-tests/update-single-check.test b/clang/test/utils/update-verify-tests/update-single-check.test
new file mode 100644
index 00000000000000..2cb1ae3bcbd3b8
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/update-single-check.test
@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/update-single-check.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/update-single-check.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
diff --git a/clang/utils/update-verify-tests.py b/clang/utils/update-verify-tests.py
index cfcfefc85e576a..f40ce15e449f22 100644
--- a/clang/utils/update-verify-tests.py
+++ b/clang/utils/update-verify-tests.py
@@ -28,6 +28,8 @@ class KnownException(Exception):
def parse_error_category(s):
+ if "no expected directives found" in line:
+ return None
parts = s.split("diagnostics")
diag_category = parts[0]
category_parts = parts[0].strip().strip("'").split("-")
@@ -64,21 +66,10 @@ def __init__(self, content, line_n):
self.content = content
self.diag = None
self.line_n = line_n
- self.related_diags = []
self.targeting_diags = []
def update_line_n(self, n):
- if self.diag and not self.diag.line_is_absolute:
- self.diag.orig_target_line_n += n - self.line_n
self.line_n = n
- for diag in self.targeting_diags:
- if diag.line_is_absolute:
- diag.orig_target_line_n = n
- else:
- diag.orig_target_line_n = n - diag.line.line_n
- for diag in self.related_diags:
- if not diag.line_is_absolute:
- pass
def render(self):
if not self.diag:
@@ -95,16 +86,17 @@ def __init__(
self,
diag_content,
category,
- targeted_line_n,
+ parsed_target_line_n,
line_is_absolute,
count,
line,
is_re,
whitespace_strings,
+ is_from_source_file,
):
self.diag_content = diag_content
self.category = category
- self.orig_target_line_n = targeted_line_n
+ self.parsed_target_line_n = parsed_target_line_n
self.line_is_absolute = line_is_absolute
self.count = count
self.line = line
@@ -112,25 +104,50 @@ def __init__(
self.is_re = is_re
self.absolute_target()
self.whitespace_strings = whitespace_strings
+ self.is_from_source_file = is_from_source_file
+
+ def decrement_count(self):
+ self.count -= 1
+ assert self.count >= 0
+
+ def increment_count(self):
+ assert self.count >= 0
+ self.count += 1
- def add(self):
- if targeted_line > 0:
- targeted_line += 1
- elif targeted_line < 0:
- targeted_line -= 1
+ def unset_target(self):
+ assert self.target is not None
+ self.target.targeting_diags.remove(self)
+ self.target = None
+
+ def set_target(self, target):
+ if self.target:
+ self.unset_target()
+ self.target = target
+ self.target.targeting_diags.append(self)
def absolute_target(self):
- if self.line_is_absolute:
- res = self.orig_target_line_n
- else:
- res = self.line.line_n + self.orig_target_line_n
if self.target:
- assert self.line.line_n == res
- return res
+ return self.target.line_n
+ if self.line_is_absolute:
+ return self.parsed_target_line_n
+ return self.line.line_n + self.parsed_target_line_n
def relative_target(self):
return self.absolute_target() - self.line.line_n
+ def take(self, other_diag):
+ assert self.count == 0
+ assert other_diag.count > 0
+ assert other_diag.target == self.target
+ assert not other_diag.line_is_absolute
+ assert not other_diag.is_re and not self.is_re
+ self.line_is_absolute = False
+ self.diag_content = other_diag.diag_content
+ self.count = other_diag.count
+ self.category = other_diag.category
+ self.count = other_diag.count
+ other_diag.count = 0
+
def render(self):
assert self.count >= 0
if self.count == 0:
@@ -151,19 +168,25 @@ def render(self):
whitespace1_s = self.whitespace_strings[0]
whitespace2_s = self.whitespace_strings[1]
whitespace3_s = self.whitespace_strings[2]
- whitespace4_s = self.whitespace_strings[3]
else:
whitespace1_s = " "
whitespace2_s = ""
whitespace3_s = ""
- whitespace4_s = ""
- if count_s and not whitespace3_s:
- whitespace3_s = " "
- return f"//{whitespace1_s}expected-{self.category}{re_s}{whitespace2_s}{line_location_s}{whitespace3_s}{count_s}{whitespace4_s}{{{{{self.diag_content}}}}}"
-
+ if count_s and not whitespace2_s:
+ whitespace2_s = " " # required to parse correctly
+ elif not count_s and whitespace2_s == " ":
+ """ Don't emit a weird extra space.
+ However if the whitespace is something other than the
+ standard single space, let it be to avoid disrupting manual formatting.
+ The existence of a non-empty whitespace2_s implies this was parsed with
+ a count > 1 and then decremented, otherwise this whitespace would have
+ been parsed as whitespace3_s.
+ """
+ whitespace2_s = ""
+ return f"//{whitespace1_s}expected-{self.category}{re_s}{line_location_s}{whitespace2_s}{count_s}{whitespace3_s}{{{{{self.diag_content}}}}}"
expected_diag_re = re.compile(
- r"//(\s*)expected-(note|warning|error)(-re)?(\s*)(@[+-]?\d+)?(\s*)(\d+)?(\s*)\{\{(.*)\}\}"
+ r"//(\s*)expected-(note|warning|error)(-re)?(@[+-]?\d+)?(?:(\s*)(\d+))?(\s*)\{\{(.*)\}\}"
)
@@ -173,19 +196,17 @@ def parse_diag(line, filename, lines):
if not ms:
return None
if len(ms) > 1:
- print(
+ raise KnownException(
f"multiple diags on line {filename}:{line.line_n}. Aborting due to missing implementation."
)
- sys.exit(1)
[
whitespace1_s,
category_s,
re_s,
- whitespace2_s,
target_line_s,
- whitespace3_s,
+ whitespace2_s,
count_s,
- whitespace4_s,
+ whitespace3_s,
diag_s,
] = ms[0]
if not target_line_s:
@@ -211,18 +232,11 @@ def parse_diag(line, filename, lines):
count,
line,
bool(re_s),
- [whitespace1_s, whitespace2_s, whitespace3_s, whitespace4_s],
+ [whitespace1_s, whitespace2_s, whitespace3_s],
+ True,
)
-def link_line_diags(lines, diag):
- line_n = diag.line.line_n
- target_line_n = diag.absolute_target()
- step = 1 if target_line_n < line_n else -1
- for i in range(target_line_n, line_n, step):
- lines[i - 1].related_diags.append(diag)
-
-
def add_line(new_line, lines):
lines.insert(new_line.line_n - 1, new_line)
for i in range(new_line.line_n, len(lines)):
@@ -231,6 +245,14 @@ def add_line(new_line, lines):
line.update_line_n(i + 1)
assert all(line.line_n == i + 1 for i, line in enumerate(lines))
+def remove_line(old_line, lines):
+ lines.remove(old_line)
+ for i in range(old_line.line_n - 1, len(lines)):
+ line = lines[i]
+ assert line.line_n == i + 2
+ line.update_line_n(i + 1)
+ assert all(line.line_n == i + 1 for i, line in enumerate(lines))
+
indent_re = re.compile(r"\s*")
@@ -238,8 +260,11 @@ def add_line(new_line, lines):
def get_indent(s):
return indent_re.match(s).group(0)
+def orig_line_n_to_new_line_n(line_n, orig_lines):
+ return orig_lines[line_n - 1].line_n
-def add_diag(line_n, diag_s, diag_category, lines):
+def add_diag(orig_line_n, diag_s, diag_category, lines, orig_lines):
+ line_n = orig_line_n_to_new_line_n(orig_line_n, orig_lines)
target = lines[line_n - 1]
for other in target.targeting_diags:
if other.is_re:
@@ -272,18 +297,42 @@ def add_diag(line_n, diag_s, diag_category, lines):
assert new_line_n == line_n + (not reverse) - total_offset
new_line = Line(get_indent(prev_line.content) + "{{DIAG}}\n", new_line_n)
- new_line.related_diags = list(prev_line.related_diags)
add_line(new_line, lines)
+ whitespace_strings = prev_line.diag.whitespace_strings if prev_line.diag else None
new_diag = Diag(
- diag_s, diag_category, total_offset, False, 1, new_line, False, None
+ diag_s, diag_category, total_offset, False, 1, new_line, False, whitespace_strings, False,
)
new_line.diag = new_diag
- new_diag.target_line = target
- assert type(new_diag) != str
- target.targeting_diags.append(new_diag)
- link_line_diags(lines, new_diag)
+ new_diag.set_target(target)
+
+def remove_dead_diags(lines):
+ for line in lines:
+ if not line.diag or line.diag.count != 0:
+ continue
+ if line.render() == "":
+ remove_line(line, lines)
+ else:
+ assert line.diag.is_from_source_file
+ for other_diag in line.targeting_diags:
+ if other_diag.is_from_source_file or other_diag.count == 0 or other_diag.category != line.diag.category:
+ continue
+ if other_diag.is_re or line.diag.is_re:
+ continue
+ line.diag.take(other_diag)
+ remove_line(other_diag.line, lines)
+
+def has_live_diags(lines):
+ for line in lines:
+ if line.diag and line.diag.count > 0:
+ return True
+ return False
+def get_expected_no_diags_line_n(lines):
+ for line in lines:
+ if "expected-no-diagnostics" in line.content:
+ return line.line_n
+ return None
updated_test_files = set()
@@ -298,13 +347,14 @@ def update_test_file(filename, diag_errors):
updated_test_files.add(filename)
with open(filename, "r") as f:
lines = [Line(line, i + 1) for i, line in enumerate(f.readlines())]
+ orig_lines = list(lines)
+ expected_no_diags_line_n = get_expected_no_diags_line_n(orig_lines)
+
for line in lines:
diag = parse_diag(line, filename, lines)
if diag:
line.diag = diag
- diag.target_line = lines[diag.absolute_target() - 1]
- link_line_diags(lines, diag)
- lines[diag.absolute_target() - 1].targeting_diags.append(diag)
+ diag.set_target(lines[diag.absolute_target() - 1])
for line_n, diag_s, diag_category, seen in diag_errors:
if seen:
@@ -319,13 +369,13 @@ def update_test_file(filename, diag_errors):
raise KnownException(
f"{filename}:{line_n} - found {lines[line_n - 1].diag.category} diag but expected {diag_category}"
)
- lines[line_n - 1].diag.count -= 1
+ lines[line_n - 1].diag.decrement_count()
diag_errors_left = []
diag_errors.sort(reverse=True, key=lambda t: t[0])
for line_n, diag_s, diag_category, seen in diag_errors:
if not seen:
continue
- target = lines[line_n - 1]
+ target = orig_lines[line_n - 1]
other_diags = [
d
for d in target.targeting_diags
@@ -333,13 +383,17 @@ def update_test_file(filename, diag_errors):
]
other_diag = other_diags[0] if other_diags else None
if other_diag:
- other_diag.count += 1
+ other_diag.increment_count()
else:
- diag_errors_left.append((line_n, diag_s, diag_category))
- for line_n, diag_s, diag_category in diag_errors_left:
- add_diag(line_n, diag_s, diag_category, lines)
+ add_diag(line_n, diag_s, diag_category, lines, orig_lines)
+ remove_dead_diags(lines)
+ has_diags = has_live_diags(lines)
with open(filename, "w") as f:
+ if not has_diags and expected_no_diags_line_n is None:
+ f.write("// expected-no-diagnostics\n")
for line in lines:
+ if has_diags and line.line_n == expected_no_diags_line_n:
+ continue
f.write(line.render())
@@ -361,6 +415,7 @@ def update_test_files(errors):
curr_category = None
curr_run_line = None
lines_since_run = []
+skip_to_next_file = False
for line in sys.stdin.readlines():
lines_since_run.append(line)
try:
@@ -374,20 +429,28 @@ def update_test_files(errors):
for line in lines_since_run:
print(line, end="")
print("====================")
- print("no mismatching diagnostics found since last RUN line")
+ if lines_since_run:
+ print("no mismatching diagnostics found since last RUN line")
+ skip_to_next_file = False
+ continue
+ if skip_to_next_file:
continue
if line.startswith("error: "):
- if "no expected directives found" in line:
- print(
- f"no expected directives found for RUN line '{curr_run_line.strip()}'. Add 'expected-no-diagnostics' manually if this is intended."
- )
- continue
curr_category = parse_error_category(line[len("error: ") :])
continue
diag_error = parse_diag_error(line.strip())
if diag_error:
curr.append((diag_error, curr_category))
+ except KnownException as e:
+ print(f"Error while parsing: {e}")
+ if curr:
+ print("skipping to next file")
+ curr = []
+ curr_category = None
+ curr_run_line = None
+ lines_since_run = []
+ skip_to_next_file = True
except Exception as e:
for line in lines_since_run:
print(line, end="")
>From dd702ce2be48dcf225755d7228b7993b698e888a Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Tue, 27 Aug 2024 17:42:50 -0700
Subject: [PATCH 3/6] [Utils] Add custom prefix support to update-verify-tests
Custom prefixes can now be provided with --prefix. The default remains
'expected'.
---
.../Inputs/non-default-prefix.c | 5 +
.../Inputs/non-default-prefix.c.expected | 5 +
.../non-default-prefix.test | 4 +
clang/utils/update-verify-tests.py | 153 ++++++++++--------
4 files changed, 98 insertions(+), 69 deletions(-)
create mode 100644 clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
create mode 100644 clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
create mode 100644 clang/test/utils/update-verify-tests/non-default-prefix.test
diff --git a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
new file mode 100644
index 00000000000000..3d63eaf0f1b878
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
@@ -0,0 +1,5 @@
+void foo() {
+ a = 2; // check-error{{asdf}}
+ // expected-error at -1{ignored}}
+}
+
diff --git a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
new file mode 100644
index 00000000000000..a877f86922123d
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
@@ -0,0 +1,5 @@
+void foo() {
+ a = 2; // check-error{{use of undeclared identifier 'a'}}
+ // expected-error at -1{ignored}}
+}
+
diff --git a/clang/test/utils/update-verify-tests/non-default-prefix.test b/clang/test/utils/update-verify-tests/non-default-prefix.test
new file mode 100644
index 00000000000000..e581755a6e6038
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/non-default-prefix.test
@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/non-default-prefix.c %t.c && not %clang_cc1 -verify=check %t.c 2>&1 | %update-verify-tests --prefix check
+# RUN: diff -u %S/Inputs/non-default-prefix.c.expected %t.c
+# RUN: %clang_cc1 -verify=check %t.c
+
diff --git a/clang/utils/update-verify-tests.py b/clang/utils/update-verify-tests.py
index f40ce15e449f22..dd83a9418ddf5f 100644
--- a/clang/utils/update-verify-tests.py
+++ b/clang/utils/update-verify-tests.py
@@ -1,5 +1,6 @@
import sys
import re
+import argparse
"""
Pipe output from clang's -verify into this script to have the test case updated to expect the actual diagnostic output.
@@ -10,7 +11,6 @@
diffs. If inaccurate their count will be updated, or the check removed entirely.
Missing features:
- - custom prefix support (-verify=my-prefix)
- multiple prefixes on the same line (-verify=my-prefix,my-other-prefix)
- multiple prefixes on separate RUN lines (RUN: -verify=my-prefix\nRUN: -verify my-other-prefix)
- regexes with expected-*-re: existing ones will be left untouched if accurate, but the script will abort if there are any
@@ -27,16 +27,16 @@ class KnownException(Exception):
pass
-def parse_error_category(s):
- if "no expected directives found" in line:
+def parse_error_category(s, prefix):
+ if "no expected directives found" in s:
return None
parts = s.split("diagnostics")
diag_category = parts[0]
category_parts = parts[0].strip().strip("'").split("-")
expected = category_parts[0]
- if expected != "expected":
+ if expected != prefix:
raise Exception(
- f"expected 'expected', but found '{expected}'. Custom verify prefixes are not supported."
+ f"expected prefix '{prefix}', but found '{expected}'. Multiple verify prefixes are not supported."
)
diag_category = category_parts[1]
if "seen but not expected" in parts[1]:
@@ -84,6 +84,7 @@ def render(self):
class Diag:
def __init__(
self,
+ prefix,
diag_content,
category,
parsed_target_line_n,
@@ -94,6 +95,7 @@ def __init__(
whitespace_strings,
is_from_source_file,
):
+ self.prefix = prefix
self.diag_content = diag_content
self.category = category
self.parsed_target_line_n = parsed_target_line_n
@@ -183,14 +185,14 @@ def render(self):
been parsed as whitespace3_s.
"""
whitespace2_s = ""
- return f"//{whitespace1_s}expected-{self.category}{re_s}{line_location_s}{whitespace2_s}{count_s}{whitespace3_s}{{{{{self.diag_content}}}}}"
+ return f"//{whitespace1_s}{self.prefix}-{self.category}{re_s}{line_location_s}{whitespace2_s}{count_s}{whitespace3_s}{{{{{self.diag_content}}}}}"
expected_diag_re = re.compile(
- r"//(\s*)expected-(note|warning|error)(-re)?(@[+-]?\d+)?(?:(\s*)(\d+))?(\s*)\{\{(.*)\}\}"
+ r"//(\s*)([a-zA-Z]+)-(note|warning|error)(-re)?(@[+-]?\d+)?(?:(\s*)(\d+))?(\s*)\{\{(.*)\}\}"
)
-def parse_diag(line, filename, lines):
+def parse_diag(line, filename, lines, prefix):
s = line.content
ms = expected_diag_re.findall(s)
if not ms:
@@ -201,6 +203,7 @@ def parse_diag(line, filename, lines):
)
[
whitespace1_s,
+ check_prefix,
category_s,
re_s,
target_line_s,
@@ -209,6 +212,8 @@ def parse_diag(line, filename, lines):
whitespace3_s,
diag_s,
] = ms[0]
+ if check_prefix != prefix:
+ return None
if not target_line_s:
target_line_n = 0
is_absolute = False
@@ -225,6 +230,7 @@ def parse_diag(line, filename, lines):
line.content = expected_diag_re.sub("{{DIAG}}", s)
return Diag(
+ prefix,
diag_s,
category_s,
target_line_n,
@@ -263,7 +269,7 @@ def get_indent(s):
def orig_line_n_to_new_line_n(line_n, orig_lines):
return orig_lines[line_n - 1].line_n
-def add_diag(orig_line_n, diag_s, diag_category, lines, orig_lines):
+def add_diag(orig_line_n, diag_s, diag_category, lines, orig_lines, prefix):
line_n = orig_line_n_to_new_line_n(orig_line_n, orig_lines)
target = lines[line_n - 1]
for other in target.targeting_diags:
@@ -301,7 +307,7 @@ def add_diag(orig_line_n, diag_s, diag_category, lines, orig_lines):
whitespace_strings = prev_line.diag.whitespace_strings if prev_line.diag else None
new_diag = Diag(
- diag_s, diag_category, total_offset, False, 1, new_line, False, whitespace_strings, False,
+ prefix, diag_s, diag_category, total_offset, False, 1, new_line, False, whitespace_strings, False,
)
new_line.diag = new_diag
new_diag.set_target(target)
@@ -328,16 +334,16 @@ def has_live_diags(lines):
return True
return False
-def get_expected_no_diags_line_n(lines):
+def get_expected_no_diags_line_n(lines, prefix):
for line in lines:
- if "expected-no-diagnostics" in line.content:
+ if f"{prefix}-no-diagnostics" in line.content:
return line.line_n
return None
updated_test_files = set()
-def update_test_file(filename, diag_errors):
+def update_test_file(filename, diag_errors, prefix):
print(f"updating test file {filename}")
if filename in updated_test_files:
print(
@@ -348,10 +354,10 @@ def update_test_file(filename, diag_errors):
with open(filename, "r") as f:
lines = [Line(line, i + 1) for i, line in enumerate(f.readlines())]
orig_lines = list(lines)
- expected_no_diags_line_n = get_expected_no_diags_line_n(orig_lines)
+ expected_no_diags_line_n = get_expected_no_diags_line_n(orig_lines, prefix)
for line in lines:
- diag = parse_diag(line, filename, lines)
+ diag = parse_diag(line, filename, lines, prefix)
if diag:
line.diag = diag
diag.set_target(lines[diag.absolute_target() - 1])
@@ -385,7 +391,7 @@ def update_test_file(filename, diag_errors):
if other_diag:
other_diag.increment_count()
else:
- add_diag(line_n, diag_s, diag_category, lines, orig_lines)
+ add_diag(line_n, diag_s, diag_category, lines, orig_lines, prefix)
remove_dead_diags(lines)
has_diags = has_live_diags(lines)
with open(filename, "w") as f:
@@ -397,7 +403,7 @@ def update_test_file(filename, diag_errors):
f.write(line.render())
-def update_test_files(errors):
+def update_test_files(errors, prefix):
errors_by_file = {}
for (filename, line, diag_s), (diag_category, seen) in errors:
if filename not in errors_by_file:
@@ -405,63 +411,72 @@ def update_test_files(errors):
errors_by_file[filename].append((line, diag_s, diag_category, seen))
for filename, diag_errors in errors_by_file.items():
try:
- update_test_file(filename, diag_errors)
+ update_test_file(filename, diag_errors, prefix)
except KnownException as e:
print(f"{filename} - ERROR: {e}")
print("continuing...")
-
-curr = []
-curr_category = None
-curr_run_line = None
-lines_since_run = []
-skip_to_next_file = False
-for line in sys.stdin.readlines():
- lines_since_run.append(line)
- try:
- if line.startswith("RUN:"):
- if curr:
- update_test_files(curr)
+def check_expectations(tool_output, prefix):
+ curr = []
+ curr_category = None
+ curr_run_line = None
+ lines_since_run = []
+ skip_to_next_file = False
+ for line in tool_output:
+ lines_since_run.append(line)
+ try:
+ if line.startswith("RUN:"):
+ if curr:
+ update_test_files(curr, prefix)
+ curr = []
+ lines_since_run = [line]
+ curr_run_line = line
+ else:
+ for line in lines_since_run:
+ print(line, end="")
+ print("====================")
+ if lines_since_run:
+ print("no mismatching diagnostics found since last RUN line")
+ skip_to_next_file = False
+ continue
+ if skip_to_next_file:
+ continue
+ if line.startswith("error: "):
+ curr_category = parse_error_category(line[len("error: ") :], prefix)
+ continue
+
+ diag_error = parse_diag_error(line.strip())
+ if diag_error:
+ curr.append((diag_error, curr_category))
+ except KnownException as e:
+ print(f"Error while parsing: {e}")
+ if curr:
+ print("skipping to next file")
curr = []
- lines_since_run = [line]
- curr_run_line = line
- else:
- for line in lines_since_run:
- print(line, end="")
- print("====================")
- if lines_since_run:
- print("no mismatching diagnostics found since last RUN line")
- skip_to_next_file = False
- continue
- if skip_to_next_file:
- continue
- if line.startswith("error: "):
- curr_category = parse_error_category(line[len("error: ") :])
- continue
-
- diag_error = parse_diag_error(line.strip())
- if diag_error:
- curr.append((diag_error, curr_category))
- except KnownException as e:
- print(f"Error while parsing: {e}")
- if curr:
- print("skipping to next file")
- curr = []
- curr_category = None
- curr_run_line = None
- lines_since_run = []
- skip_to_next_file = True
- except Exception as e:
+ curr_category = None
+ curr_run_line = None
+ lines_since_run = []
+ skip_to_next_file = True
+ except Exception as e:
+ for line in lines_since_run:
+ print(line, end="")
+ print("====================")
+ print(e)
+ sys.exit(1)
+ if curr:
+ update_test_files(curr, prefix)
+ print("done!")
+ else:
for line in lines_since_run:
print(line, end="")
print("====================")
- print(e)
- sys.exit(1)
-if curr:
- update_test_files(curr)
- print("done!")
-else:
- for line in lines_since_run:
- print(line, end="")
- print("====================")
- print("no mismatching diagnostics found")
+ print("no mismatching diagnostics found")
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--prefix", default="expected", help="The prefix passed to -verify")
+ args = parser.parse_args()
+ check_expectations(sys.stdin.readlines(), args.prefix)
+
+if __name__ == "__main__":
+ main()
>From 7395ef2db8c43f41cdbed70a4fa8c06579a078fe Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 11 Sep 2024 17:05:43 -0700
Subject: [PATCH 4/6] [Utils] Separate update-verify-tests script from core
The script no longer attempts to be RUN line aware. Feeding raw llvm-lit
output into it _may_ still work, but given a large enough number of
test cases to update, the risk that some test case uses unsupported
features grows pretty large. This will instead be handled by adding a
lit integration to automatically invoke the core for each failing test
output. The lit integration will be landed as a separate change.
---
clang/utils/UpdateVerifyTests/core.py | 452 +++++++++++++++++++++++++
clang/utils/update-verify-tests.py | 462 +-------------------------
2 files changed, 461 insertions(+), 453 deletions(-)
create mode 100644 clang/utils/UpdateVerifyTests/core.py
diff --git a/clang/utils/UpdateVerifyTests/core.py b/clang/utils/UpdateVerifyTests/core.py
new file mode 100644
index 00000000000000..d1350cdbb698b6
--- /dev/null
+++ b/clang/utils/UpdateVerifyTests/core.py
@@ -0,0 +1,452 @@
+import sys
+import re
+
+DEBUG = False
+
+
+def dprint(*args):
+ if DEBUG:
+ print(*args, file=sys.stderr)
+
+
+class KnownException(Exception):
+ pass
+
+
+def parse_error_category(s, prefix):
+ if "no expected directives found" in s:
+ return None
+ parts = s.split("diagnostics")
+ diag_category = parts[0]
+ category_parts = parts[0].strip().strip("'").split("-")
+ expected = category_parts[0]
+ if expected != prefix:
+ raise Exception(
+ f"expected prefix '{prefix}', but found '{expected}'. Multiple verify prefixes are not supported."
+ )
+ diag_category = category_parts[1]
+ if "seen but not expected" in parts[1]:
+ seen = True
+ elif "expected but not seen" in parts[1]:
+ seen = False
+ else:
+ raise KnownException(f"unexpected category '{parts[1]}'")
+ return (diag_category, seen)
+
+
+diag_error_re = re.compile(r"File (\S+) Line (\d+): (.+)")
+diag_error_re2 = re.compile(r"File \S+ Line \d+ \(directive at (\S+):(\d+)\): (.+)")
+
+
+def parse_diag_error(s):
+ m = diag_error_re2.match(s)
+ if not m:
+ m = diag_error_re.match(s)
+ if not m:
+ return None
+ return (m.group(1), int(m.group(2)), m.group(3))
+
+
+class Line:
+ def __init__(self, content, line_n):
+ self.content = content
+ self.diag = None
+ self.line_n = line_n
+ self.targeting_diags = []
+
+ def update_line_n(self, n):
+ self.line_n = n
+
+ def render(self):
+ if not self.diag:
+ return self.content
+ assert "{{DIAG}}" in self.content
+ res = self.content.replace("{{DIAG}}", self.diag.render())
+ if not res.strip():
+ return ""
+ return res
+
+
+class Diag:
+ def __init__(
+ self,
+ prefix,
+ diag_content,
+ category,
+ parsed_target_line_n,
+ line_is_absolute,
+ count,
+ line,
+ is_re,
+ whitespace_strings,
+ is_from_source_file,
+ ):
+ self.prefix = prefix
+ self.diag_content = diag_content
+ self.category = category
+ self.parsed_target_line_n = parsed_target_line_n
+ self.line_is_absolute = line_is_absolute
+ self.count = count
+ self.line = line
+ self.target = None
+ self.is_re = is_re
+ self.absolute_target()
+ self.whitespace_strings = whitespace_strings
+ self.is_from_source_file = is_from_source_file
+
+ def decrement_count(self):
+ self.count -= 1
+ assert self.count >= 0
+
+ def increment_count(self):
+ assert self.count >= 0
+ self.count += 1
+
+ def unset_target(self):
+ assert self.target is not None
+ self.target.targeting_diags.remove(self)
+ self.target = None
+
+ def set_target(self, target):
+ if self.target:
+ self.unset_target()
+ self.target = target
+ self.target.targeting_diags.append(self)
+
+ def absolute_target(self):
+ if self.target:
+ return self.target.line_n
+ if self.line_is_absolute:
+ return self.parsed_target_line_n
+ return self.line.line_n + self.parsed_target_line_n
+
+ def relative_target(self):
+ return self.absolute_target() - self.line.line_n
+
+ def take(self, other_diag):
+ assert self.count == 0
+ assert other_diag.count > 0
+ assert other_diag.target == self.target
+ assert not other_diag.line_is_absolute
+ assert not other_diag.is_re and not self.is_re
+ self.line_is_absolute = False
+ self.diag_content = other_diag.diag_content
+ self.count = other_diag.count
+ self.category = other_diag.category
+ self.count = other_diag.count
+ other_diag.count = 0
+
+ def render(self):
+ assert self.count >= 0
+ if self.count == 0:
+ return ""
+ line_location_s = ""
+ if self.relative_target() != 0:
+ if self.line_is_absolute:
+ line_location_s = f"@{self.absolute_target()}"
+ elif self.relative_target() > 0:
+ line_location_s = f"@+{self.relative_target()}"
+ else:
+ line_location_s = (
+ f"@{self.relative_target()}" # the minus sign is implicit
+ )
+ count_s = "" if self.count == 1 else f"{self.count}"
+ re_s = "-re" if self.is_re else ""
+ if self.whitespace_strings:
+ whitespace1_s = self.whitespace_strings[0]
+ whitespace2_s = self.whitespace_strings[1]
+ whitespace3_s = self.whitespace_strings[2]
+ else:
+ whitespace1_s = " "
+ whitespace2_s = ""
+ whitespace3_s = ""
+ if count_s and not whitespace2_s:
+ whitespace2_s = " " # required to parse correctly
+ elif not count_s and whitespace2_s == " ":
+ """Don't emit a weird extra space.
+ However if the whitespace is something other than the
+ standard single space, let it be to avoid disrupting manual formatting.
+ The existence of a non-empty whitespace2_s implies this was parsed with
+ a count > 1 and then decremented, otherwise this whitespace would have
+ been parsed as whitespace3_s.
+ """
+ whitespace2_s = ""
+ return f"//{whitespace1_s}{self.prefix}-{self.category}{re_s}{line_location_s}{whitespace2_s}{count_s}{whitespace3_s}{{{{{self.diag_content}}}}}"
+
+
+expected_diag_re = re.compile(
+ r"//(\s*)([a-zA-Z]+)-(note|warning|error)(-re)?(@[+-]?\d+)?(?:(\s*)(\d+))?(\s*)\{\{(.*)\}\}"
+)
+
+
+def parse_diag(line, filename, lines, prefix):
+ s = line.content
+ ms = expected_diag_re.findall(s)
+ if not ms:
+ return None
+ if len(ms) > 1:
+ raise KnownException(
+ f"multiple diags on line {filename}:{line.line_n}. Aborting due to missing implementation."
+ )
+ [
+ whitespace1_s,
+ check_prefix,
+ category_s,
+ re_s,
+ target_line_s,
+ whitespace2_s,
+ count_s,
+ whitespace3_s,
+ diag_s,
+ ] = ms[0]
+ if check_prefix != prefix:
+ return None
+ if not target_line_s:
+ target_line_n = 0
+ is_absolute = False
+ elif target_line_s.startswith("@+"):
+ target_line_n = int(target_line_s[2:])
+ is_absolute = False
+ elif target_line_s.startswith("@-"):
+ target_line_n = int(target_line_s[1:])
+ is_absolute = False
+ else:
+ target_line_n = int(target_line_s[1:])
+ is_absolute = True
+ count = int(count_s) if count_s else 1
+ line.content = expected_diag_re.sub("{{DIAG}}", s)
+
+ return Diag(
+ prefix,
+ diag_s,
+ category_s,
+ target_line_n,
+ is_absolute,
+ count,
+ line,
+ bool(re_s),
+ [whitespace1_s, whitespace2_s, whitespace3_s],
+ True,
+ )
+
+
+def add_line(new_line, lines):
+ lines.insert(new_line.line_n - 1, new_line)
+ for i in range(new_line.line_n, len(lines)):
+ line = lines[i]
+ assert line.line_n == i
+ line.update_line_n(i + 1)
+ assert all(line.line_n == i + 1 for i, line in enumerate(lines))
+
+
+def remove_line(old_line, lines):
+ lines.remove(old_line)
+ for i in range(old_line.line_n - 1, len(lines)):
+ line = lines[i]
+ assert line.line_n == i + 2
+ line.update_line_n(i + 1)
+ assert all(line.line_n == i + 1 for i, line in enumerate(lines))
+
+
+indent_re = re.compile(r"\s*")
+
+
+def get_indent(s):
+ return indent_re.match(s).group(0)
+
+
+def orig_line_n_to_new_line_n(line_n, orig_lines):
+ return orig_lines[line_n - 1].line_n
+
+
+def add_diag(orig_line_n, diag_s, diag_category, lines, orig_lines, prefix):
+ line_n = orig_line_n_to_new_line_n(orig_line_n, orig_lines)
+ target = lines[line_n - 1]
+ for other in target.targeting_diags:
+ if other.is_re:
+ raise KnownException(
+ "mismatching diag on line with regex matcher. Skipping due to missing implementation"
+ )
+ reverse = (
+ True
+ if [other for other in target.targeting_diags if other.relative_target() < 0]
+ else False
+ )
+
+ targeting = [
+ other for other in target.targeting_diags if not other.line_is_absolute
+ ]
+ targeting.sort(reverse=reverse, key=lambda d: d.relative_target())
+ prev_offset = 0
+ prev_line = target
+ direction = -1 if reverse else 1
+ for d in targeting:
+ if d.relative_target() != prev_offset + direction:
+ break
+ prev_offset = d.relative_target()
+ prev_line = d.line
+ total_offset = prev_offset - 1 if reverse else prev_offset + 1
+ if reverse:
+ new_line_n = prev_line.line_n + 1
+ else:
+ new_line_n = prev_line.line_n
+ assert new_line_n == line_n + (not reverse) - total_offset
+
+ new_line = Line(get_indent(prev_line.content) + "{{DIAG}}\n", new_line_n)
+ add_line(new_line, lines)
+
+ whitespace_strings = prev_line.diag.whitespace_strings if prev_line.diag else None
+ new_diag = Diag(
+ prefix,
+ diag_s,
+ diag_category,
+ total_offset,
+ False,
+ 1,
+ new_line,
+ False,
+ whitespace_strings,
+ False,
+ )
+ new_line.diag = new_diag
+ new_diag.set_target(target)
+
+
+def remove_dead_diags(lines):
+ for line in lines:
+ if not line.diag or line.diag.count != 0:
+ continue
+ if line.render() == "":
+ remove_line(line, lines)
+ else:
+ assert line.diag.is_from_source_file
+ for other_diag in line.targeting_diags:
+ if (
+ other_diag.is_from_source_file
+ or other_diag.count == 0
+ or other_diag.category != line.diag.category
+ ):
+ continue
+ if other_diag.is_re or line.diag.is_re:
+ continue
+ line.diag.take(other_diag)
+ remove_line(other_diag.line, lines)
+
+
+def has_live_diags(lines):
+ for line in lines:
+ if line.diag and line.diag.count > 0:
+ return True
+ return False
+
+
+def get_expected_no_diags_line_n(lines, prefix):
+ for line in lines:
+ if f"{prefix}-no-diagnostics" in line.content:
+ return line.line_n
+ return None
+
+
+def update_test_file(filename, diag_errors, prefix, updated_test_files):
+ dprint(f"updating test file {filename}")
+ if filename in updated_test_files:
+ raise KnownException(f"{filename} already updated, but got new output")
+ else:
+ updated_test_files.add(filename)
+ with open(filename, "r") as f:
+ lines = [Line(line, i + 1) for i, line in enumerate(f.readlines())]
+ orig_lines = list(lines)
+ expected_no_diags_line_n = get_expected_no_diags_line_n(orig_lines, prefix)
+
+ for line in lines:
+ diag = parse_diag(line, filename, lines, prefix)
+ if diag:
+ line.diag = diag
+ diag.set_target(lines[diag.absolute_target() - 1])
+
+ for line_n, diag_s, diag_category, seen in diag_errors:
+ if seen:
+ continue
+ # this is a diagnostic expected but not seen
+ assert lines[line_n - 1].diag
+ if diag_s != lines[line_n - 1].diag.diag_content:
+ raise KnownException(
+ f"{filename}:{line_n} - found diag {lines[line_n - 1].diag.diag_content} but expected {diag_s}"
+ )
+ if diag_category != lines[line_n - 1].diag.category:
+ raise KnownException(
+ f"{filename}:{line_n} - found {lines[line_n - 1].diag.category} diag but expected {diag_category}"
+ )
+ lines[line_n - 1].diag.decrement_count()
+ diag_errors_left = []
+ diag_errors.sort(reverse=True, key=lambda t: t[0])
+ for line_n, diag_s, diag_category, seen in diag_errors:
+ if not seen:
+ continue
+ target = orig_lines[line_n - 1]
+ other_diags = [
+ d
+ for d in target.targeting_diags
+ if d.diag_content == diag_s and d.category == diag_category
+ ]
+ other_diag = other_diags[0] if other_diags else None
+ if other_diag:
+ other_diag.increment_count()
+ else:
+ add_diag(line_n, diag_s, diag_category, lines, orig_lines, prefix)
+ remove_dead_diags(lines)
+ has_diags = has_live_diags(lines)
+ with open(filename, "w") as f:
+ if not has_diags and expected_no_diags_line_n is None:
+ f.write("// expected-no-diagnostics\n")
+ for line in lines:
+ if has_diags and line.line_n == expected_no_diags_line_n:
+ continue
+ f.write(line.render())
+
+
+def update_test_files(errors, prefix):
+ errors_by_file = {}
+ for (filename, line, diag_s), (diag_category, seen) in errors:
+ if filename not in errors_by_file:
+ errors_by_file[filename] = []
+ errors_by_file[filename].append((line, diag_s, diag_category, seen))
+ updated_test_files = set()
+ for filename, diag_errors in errors_by_file.items():
+ try:
+ update_test_file(filename, diag_errors, prefix, updated_test_files)
+ except KnownException as e:
+ return f"Error in update-verify-tests while updating {filename}: {e}"
+ updated_files = list(updated_test_files)
+ assert updated_files
+ if len(updated_files) == 1:
+ return f"updated file {updated_files[0]}"
+ updated_files_s = "\n\t".join(updated_files)
+ return "updated files:\n\t{updated_files_s}"
+
+
+def check_expectations(tool_output, prefix):
+ """
+ The entry point function.
+ Called by the stand-alone update-verify-tests.py as well as litplugin.py.
+ """
+ curr = []
+ curr_category = None
+ try:
+ for line in tool_output:
+ if line.startswith("error: "):
+ curr_category = parse_error_category(line[len("error: ") :], prefix)
+ continue
+
+ diag_error = parse_diag_error(line.strip())
+ if diag_error:
+ curr.append((diag_error, curr_category))
+ else:
+ dprint("no match")
+ dprint(line.strip())
+ except KnownException as e:
+ return f"Error in update-verify-tests while parsing tool output: {e}"
+ if curr:
+ return update_test_files(curr, prefix)
+ else:
+ return "no mismatching diagnostics found"
diff --git a/clang/utils/update-verify-tests.py b/clang/utils/update-verify-tests.py
index dd83a9418ddf5f..e2874a8c049ef3 100644
--- a/clang/utils/update-verify-tests.py
+++ b/clang/utils/update-verify-tests.py
@@ -1,6 +1,6 @@
import sys
-import re
import argparse
+from UpdateVerifyTests.core import check_expectations
"""
Pipe output from clang's -verify into this script to have the test case updated to expect the actual diagnostic output.
@@ -19,464 +19,20 @@
- if multiple checks targeting the same line are failing the script is not guaranteed to produce a minimal diff
Example usage:
- build/bin/llvm-lit clang/test/Sema/ --no-progress-bar -v | python3 update-verify-tests.py
+ clang -verify [file] | python3 update-verify-tests.py
+ clang -verify=check [file] | python3 update-verify-tests.py --prefix check
"""
-class KnownException(Exception):
- pass
-
-
-def parse_error_category(s, prefix):
- if "no expected directives found" in s:
- return None
- parts = s.split("diagnostics")
- diag_category = parts[0]
- category_parts = parts[0].strip().strip("'").split("-")
- expected = category_parts[0]
- if expected != prefix:
- raise Exception(
- f"expected prefix '{prefix}', but found '{expected}'. Multiple verify prefixes are not supported."
- )
- diag_category = category_parts[1]
- if "seen but not expected" in parts[1]:
- seen = True
- elif "expected but not seen" in parts[1]:
- seen = False
- else:
- raise KnownException(f"unexpected category '{parts[1]}'")
- return (diag_category, seen)
-
-
-diag_error_re = re.compile(r"File (\S+) Line (\d+): (.+)")
-diag_error_re2 = re.compile(r"File \S+ Line \d+ \(directive at (\S+):(\d+)\): (.+)")
-
-
-def parse_diag_error(s):
- m = diag_error_re2.match(s)
- if not m:
- m = diag_error_re.match(s)
- if not m:
- return None
- return (m.group(1), int(m.group(2)), m.group(3))
-
-
-class Line:
- def __init__(self, content, line_n):
- self.content = content
- self.diag = None
- self.line_n = line_n
- self.targeting_diags = []
-
- def update_line_n(self, n):
- self.line_n = n
-
- def render(self):
- if not self.diag:
- return self.content
- assert "{{DIAG}}" in self.content
- res = self.content.replace("{{DIAG}}", self.diag.render())
- if not res.strip():
- return ""
- return res
-
-
-class Diag:
- def __init__(
- self,
- prefix,
- diag_content,
- category,
- parsed_target_line_n,
- line_is_absolute,
- count,
- line,
- is_re,
- whitespace_strings,
- is_from_source_file,
- ):
- self.prefix = prefix
- self.diag_content = diag_content
- self.category = category
- self.parsed_target_line_n = parsed_target_line_n
- self.line_is_absolute = line_is_absolute
- self.count = count
- self.line = line
- self.target = None
- self.is_re = is_re
- self.absolute_target()
- self.whitespace_strings = whitespace_strings
- self.is_from_source_file = is_from_source_file
-
- def decrement_count(self):
- self.count -= 1
- assert self.count >= 0
-
- def increment_count(self):
- assert self.count >= 0
- self.count += 1
-
- def unset_target(self):
- assert self.target is not None
- self.target.targeting_diags.remove(self)
- self.target = None
-
- def set_target(self, target):
- if self.target:
- self.unset_target()
- self.target = target
- self.target.targeting_diags.append(self)
-
- def absolute_target(self):
- if self.target:
- return self.target.line_n
- if self.line_is_absolute:
- return self.parsed_target_line_n
- return self.line.line_n + self.parsed_target_line_n
-
- def relative_target(self):
- return self.absolute_target() - self.line.line_n
-
- def take(self, other_diag):
- assert self.count == 0
- assert other_diag.count > 0
- assert other_diag.target == self.target
- assert not other_diag.line_is_absolute
- assert not other_diag.is_re and not self.is_re
- self.line_is_absolute = False
- self.diag_content = other_diag.diag_content
- self.count = other_diag.count
- self.category = other_diag.category
- self.count = other_diag.count
- other_diag.count = 0
-
- def render(self):
- assert self.count >= 0
- if self.count == 0:
- return ""
- line_location_s = ""
- if self.relative_target() != 0:
- if self.line_is_absolute:
- line_location_s = f"@{self.absolute_target()}"
- elif self.relative_target() > 0:
- line_location_s = f"@+{self.relative_target()}"
- else:
- line_location_s = (
- f"@{self.relative_target()}" # the minus sign is implicit
- )
- count_s = "" if self.count == 1 else f"{self.count}"
- re_s = "-re" if self.is_re else ""
- if self.whitespace_strings:
- whitespace1_s = self.whitespace_strings[0]
- whitespace2_s = self.whitespace_strings[1]
- whitespace3_s = self.whitespace_strings[2]
- else:
- whitespace1_s = " "
- whitespace2_s = ""
- whitespace3_s = ""
- if count_s and not whitespace2_s:
- whitespace2_s = " " # required to parse correctly
- elif not count_s and whitespace2_s == " ":
- """ Don't emit a weird extra space.
- However if the whitespace is something other than the
- standard single space, let it be to avoid disrupting manual formatting.
- The existence of a non-empty whitespace2_s implies this was parsed with
- a count > 1 and then decremented, otherwise this whitespace would have
- been parsed as whitespace3_s.
- """
- whitespace2_s = ""
- return f"//{whitespace1_s}{self.prefix}-{self.category}{re_s}{line_location_s}{whitespace2_s}{count_s}{whitespace3_s}{{{{{self.diag_content}}}}}"
-
-expected_diag_re = re.compile(
- r"//(\s*)([a-zA-Z]+)-(note|warning|error)(-re)?(@[+-]?\d+)?(?:(\s*)(\d+))?(\s*)\{\{(.*)\}\}"
-)
-
-
-def parse_diag(line, filename, lines, prefix):
- s = line.content
- ms = expected_diag_re.findall(s)
- if not ms:
- return None
- if len(ms) > 1:
- raise KnownException(
- f"multiple diags on line {filename}:{line.line_n}. Aborting due to missing implementation."
- )
- [
- whitespace1_s,
- check_prefix,
- category_s,
- re_s,
- target_line_s,
- whitespace2_s,
- count_s,
- whitespace3_s,
- diag_s,
- ] = ms[0]
- if check_prefix != prefix:
- return None
- if not target_line_s:
- target_line_n = 0
- is_absolute = False
- elif target_line_s.startswith("@+"):
- target_line_n = int(target_line_s[2:])
- is_absolute = False
- elif target_line_s.startswith("@-"):
- target_line_n = int(target_line_s[1:])
- is_absolute = False
- else:
- target_line_n = int(target_line_s[1:])
- is_absolute = True
- count = int(count_s) if count_s else 1
- line.content = expected_diag_re.sub("{{DIAG}}", s)
-
- return Diag(
- prefix,
- diag_s,
- category_s,
- target_line_n,
- is_absolute,
- count,
- line,
- bool(re_s),
- [whitespace1_s, whitespace2_s, whitespace3_s],
- True,
- )
-
-
-def add_line(new_line, lines):
- lines.insert(new_line.line_n - 1, new_line)
- for i in range(new_line.line_n, len(lines)):
- line = lines[i]
- assert line.line_n == i
- line.update_line_n(i + 1)
- assert all(line.line_n == i + 1 for i, line in enumerate(lines))
-
-def remove_line(old_line, lines):
- lines.remove(old_line)
- for i in range(old_line.line_n - 1, len(lines)):
- line = lines[i]
- assert line.line_n == i + 2
- line.update_line_n(i + 1)
- assert all(line.line_n == i + 1 for i, line in enumerate(lines))
-
-
-indent_re = re.compile(r"\s*")
-
-
-def get_indent(s):
- return indent_re.match(s).group(0)
-
-def orig_line_n_to_new_line_n(line_n, orig_lines):
- return orig_lines[line_n - 1].line_n
-
-def add_diag(orig_line_n, diag_s, diag_category, lines, orig_lines, prefix):
- line_n = orig_line_n_to_new_line_n(orig_line_n, orig_lines)
- target = lines[line_n - 1]
- for other in target.targeting_diags:
- if other.is_re:
- raise KnownException(
- "mismatching diag on line with regex matcher. Skipping due to missing implementation"
- )
- reverse = (
- True
- if [other for other in target.targeting_diags if other.relative_target() < 0]
- else False
- )
-
- targeting = [
- other for other in target.targeting_diags if not other.line_is_absolute
- ]
- targeting.sort(reverse=reverse, key=lambda d: d.relative_target())
- prev_offset = 0
- prev_line = target
- direction = -1 if reverse else 1
- for d in targeting:
- if d.relative_target() != prev_offset + direction:
- break
- prev_offset = d.relative_target()
- prev_line = d.line
- total_offset = prev_offset - 1 if reverse else prev_offset + 1
- if reverse:
- new_line_n = prev_line.line_n + 1
- else:
- new_line_n = prev_line.line_n
- assert new_line_n == line_n + (not reverse) - total_offset
-
- new_line = Line(get_indent(prev_line.content) + "{{DIAG}}\n", new_line_n)
- add_line(new_line, lines)
-
- whitespace_strings = prev_line.diag.whitespace_strings if prev_line.diag else None
- new_diag = Diag(
- prefix, diag_s, diag_category, total_offset, False, 1, new_line, False, whitespace_strings, False,
- )
- new_line.diag = new_diag
- new_diag.set_target(target)
-
-def remove_dead_diags(lines):
- for line in lines:
- if not line.diag or line.diag.count != 0:
- continue
- if line.render() == "":
- remove_line(line, lines)
- else:
- assert line.diag.is_from_source_file
- for other_diag in line.targeting_diags:
- if other_diag.is_from_source_file or other_diag.count == 0 or other_diag.category != line.diag.category:
- continue
- if other_diag.is_re or line.diag.is_re:
- continue
- line.diag.take(other_diag)
- remove_line(other_diag.line, lines)
-
-def has_live_diags(lines):
- for line in lines:
- if line.diag and line.diag.count > 0:
- return True
- return False
-
-def get_expected_no_diags_line_n(lines, prefix):
- for line in lines:
- if f"{prefix}-no-diagnostics" in line.content:
- return line.line_n
- return None
-
-updated_test_files = set()
-
-
-def update_test_file(filename, diag_errors, prefix):
- print(f"updating test file {filename}")
- if filename in updated_test_files:
- print(
- f"{filename} already updated, but got new output - expect incorrect results"
- )
- else:
- updated_test_files.add(filename)
- with open(filename, "r") as f:
- lines = [Line(line, i + 1) for i, line in enumerate(f.readlines())]
- orig_lines = list(lines)
- expected_no_diags_line_n = get_expected_no_diags_line_n(orig_lines, prefix)
-
- for line in lines:
- diag = parse_diag(line, filename, lines, prefix)
- if diag:
- line.diag = diag
- diag.set_target(lines[diag.absolute_target() - 1])
-
- for line_n, diag_s, diag_category, seen in diag_errors:
- if seen:
- continue
- # this is a diagnostic expected but not seen
- assert lines[line_n - 1].diag
- if diag_s != lines[line_n - 1].diag.diag_content:
- raise KnownException(
- f"{filename}:{line_n} - found diag {lines[line_n - 1].diag.diag_content} but expected {diag_s}"
- )
- if diag_category != lines[line_n - 1].diag.category:
- raise KnownException(
- f"{filename}:{line_n} - found {lines[line_n - 1].diag.category} diag but expected {diag_category}"
- )
- lines[line_n - 1].diag.decrement_count()
- diag_errors_left = []
- diag_errors.sort(reverse=True, key=lambda t: t[0])
- for line_n, diag_s, diag_category, seen in diag_errors:
- if not seen:
- continue
- target = orig_lines[line_n - 1]
- other_diags = [
- d
- for d in target.targeting_diags
- if d.diag_content == diag_s and d.category == diag_category
- ]
- other_diag = other_diags[0] if other_diags else None
- if other_diag:
- other_diag.increment_count()
- else:
- add_diag(line_n, diag_s, diag_category, lines, orig_lines, prefix)
- remove_dead_diags(lines)
- has_diags = has_live_diags(lines)
- with open(filename, "w") as f:
- if not has_diags and expected_no_diags_line_n is None:
- f.write("// expected-no-diagnostics\n")
- for line in lines:
- if has_diags and line.line_n == expected_no_diags_line_n:
- continue
- f.write(line.render())
-
-
-def update_test_files(errors, prefix):
- errors_by_file = {}
- for (filename, line, diag_s), (diag_category, seen) in errors:
- if filename not in errors_by_file:
- errors_by_file[filename] = []
- errors_by_file[filename].append((line, diag_s, diag_category, seen))
- for filename, diag_errors in errors_by_file.items():
- try:
- update_test_file(filename, diag_errors, prefix)
- except KnownException as e:
- print(f"{filename} - ERROR: {e}")
- print("continuing...")
-
-def check_expectations(tool_output, prefix):
- curr = []
- curr_category = None
- curr_run_line = None
- lines_since_run = []
- skip_to_next_file = False
- for line in tool_output:
- lines_since_run.append(line)
- try:
- if line.startswith("RUN:"):
- if curr:
- update_test_files(curr, prefix)
- curr = []
- lines_since_run = [line]
- curr_run_line = line
- else:
- for line in lines_since_run:
- print(line, end="")
- print("====================")
- if lines_since_run:
- print("no mismatching diagnostics found since last RUN line")
- skip_to_next_file = False
- continue
- if skip_to_next_file:
- continue
- if line.startswith("error: "):
- curr_category = parse_error_category(line[len("error: ") :], prefix)
- continue
-
- diag_error = parse_diag_error(line.strip())
- if diag_error:
- curr.append((diag_error, curr_category))
- except KnownException as e:
- print(f"Error while parsing: {e}")
- if curr:
- print("skipping to next file")
- curr = []
- curr_category = None
- curr_run_line = None
- lines_since_run = []
- skip_to_next_file = True
- except Exception as e:
- for line in lines_since_run:
- print(line, end="")
- print("====================")
- print(e)
- sys.exit(1)
- if curr:
- update_test_files(curr, prefix)
- print("done!")
- else:
- for line in lines_since_run:
- print(line, end="")
- print("====================")
- print("no mismatching diagnostics found")
-
def main():
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--prefix", default="expected", help="The prefix passed to -verify")
+ parser.add_argument(
+ "--prefix", default="expected", help="The prefix passed to -verify"
+ )
args = parser.parse_args()
- check_expectations(sys.stdin.readlines(), args.prefix)
+ output = check_expectations(sys.stdin.readlines(), args.prefix)
+ print(output)
+
if __name__ == "__main__":
main()
>From 77a9a8a09d20aac15596da975139c161e9bf6194 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Wed, 28 Aug 2024 23:30:49 -0700
Subject: [PATCH 5/6] [Utils] Add --update-tests to lit
This adds a flag to lit for detecting and updating failing tests when
possible to do so automatically. The flag uses a plugin architecture
where config files can add additional auto-updaters for the types of
tests in the test suite. When a test fails with --update-tests enabled
lit passes the test RUN invocation and output to each registered test
updater until one of them signals that it updated the test. As such it
is the responsibility of the test updater to only update tests where it
is reasonably certain that it will actually fix the test, or come close
to doing so.
Also adds one initial test updater specific to clang's Sema test suite,
which updates the expected-[diag-kind] lines in tests using the -verify
flag.
---
clang/test/Sema/lit.local.cfg | 12 +++++++
.../Inputs/duplicate-diag.c | 2 ++
.../Inputs/duplicate-diag.c.expected | 2 ++
.../Inputs/infer-indentation.c | 2 ++
.../Inputs/infer-indentation.c.expected | 2 ++
.../Inputs/leave-existing-diags.c | 4 ++-
.../Inputs/leave-existing-diags.c.expected | 4 ++-
.../Inputs/multiple-errors.c | 2 ++
.../Inputs/multiple-errors.c.expected | 2 ++
.../multiple-missing-errors-same-line.c | 2 ++
...ltiple-missing-errors-same-line.c.expected | 2 ++
.../update-verify-tests/Inputs/no-checks.c | 2 ++
.../Inputs/no-checks.c.expected | 2 ++
.../update-verify-tests/Inputs/no-diags.c | 2 ++
.../Inputs/no-diags.c.expected | 2 ++
.../Inputs/no-expected-diags.c | 2 ++
.../Inputs/no-expected-diags.c.expected | 2 ++
.../Inputs/non-default-prefix.c | 2 ++
.../Inputs/non-default-prefix.c.expected | 2 ++
.../Inputs/update-same-line.c | 2 ++
.../Inputs/update-same-line.c.expected | 2 ++
.../Inputs/update-single-check.c | 2 ++
.../Inputs/update-single-check.c.expected | 2 ++
.../update-verify-tests/LitTests/.gitignore | 4 +++
.../update-verify-tests/LitTests/lit.cfg | 31 ++++++++++++++++
.../utils/update-verify-tests/lit-plugin.test | 5 +++
.../utils/update-verify-tests/lit.local.cfg | 11 ++++++
clang/utils/UpdateVerifyTests/__init__.py | 1 +
clang/utils/UpdateVerifyTests/litplugin.py | 35 +++++++++++++++++++
clang/utils/update-verify-tests.py | 3 ++
llvm/utils/lit/lit/LitConfig.py | 3 ++
llvm/utils/lit/lit/TestRunner.py | 12 +++++++
llvm/utils/lit/lit/cl_arguments.py | 6 ++++
llvm/utils/lit/lit/llvm/config.py | 5 +++
llvm/utils/lit/lit/main.py | 1 +
35 files changed, 175 insertions(+), 2 deletions(-)
create mode 100644 clang/test/utils/update-verify-tests/LitTests/.gitignore
create mode 100644 clang/test/utils/update-verify-tests/LitTests/lit.cfg
create mode 100644 clang/test/utils/update-verify-tests/lit-plugin.test
create mode 100644 clang/utils/UpdateVerifyTests/__init__.py
create mode 100644 clang/utils/UpdateVerifyTests/litplugin.py
diff --git a/clang/test/Sema/lit.local.cfg b/clang/test/Sema/lit.local.cfg
index baf1b39ef238cd..b385fd92098a8e 100644
--- a/clang/test/Sema/lit.local.cfg
+++ b/clang/test/Sema/lit.local.cfg
@@ -2,3 +2,15 @@ config.substitutions = list(config.substitutions)
config.substitutions.insert(
0, (r"%clang\b", """*** Do not use the driver in Sema tests. ***""")
)
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ curdir = os.path.dirname(os.path.realpath(__file__))
+ testdir = os.path.dirname(curdir)
+ clangdir = os.path.dirname(testdir)
+ utilspath = os.path.join(clangdir, "utils")
+ sys.path.append(utilspath)
+ from UpdateVerifyTests.litplugin import verify_test_updater
+ lit_config.test_updaters.append(verify_test_updater)
diff --git a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
index 8c7e46c6eca9c1..d4a92eb4a7874a 100644
--- a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
+++ b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'a'}}
a = 2; a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
index 6214ff382f4495..5fd72709e487f5 100644
--- a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1 2{{use of undeclared identifier 'a'}}
a = 2; a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
index 0210ac35fd5cd1..9a58244d92e066 100644
--- a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
+++ b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1 2 {{use of undeclared identifier 'a'}}
a = 2; a = 2; b = 2; b = 2; c = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
index 5c5aaeeef97acf..127efb003b1883 100644
--- a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +3 {{use of undeclared identifier 'c'}}
// expected-error at +2 2 {{use of undeclared identifier 'b'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
index 1aa8d088e97273..075d6c80fd4c36 100644
--- a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
+++ b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
a = 2;
// expected-error at -1{{use of undeclared identifier 'a'}}
b = 2;// expected-error{{use of undeclared identifier 'b'}}
c = 2;
- // expected-error at 5{{use of undeclared identifier 'c'}}
+ // expected-error at 7{{use of undeclared identifier 'c'}}
d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
e = 2; // error to trigger mismatch
diff --git a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
index 6b621061bbfbbd..778ebd4ff4c7b9 100644
--- a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
a = 2;
// expected-error at -1{{use of undeclared identifier 'a'}}
b = 2;// expected-error{{use of undeclared identifier 'b'}}
c = 2;
- // expected-error at 5{{use of undeclared identifier 'c'}}
+ // expected-error at 7{{use of undeclared identifier 'c'}}
d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
// expected-error at +1{{use of undeclared identifier 'e'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
index e230e0a337bf49..d3fa93ec0e8221 100644
--- a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
a = 2;
b = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
index 27dc1f30a26faf..aa0cf22c8e8a7b 100644
--- a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'a'}}
a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
index 03f723d44bbe82..9c5104e693ae57 100644
--- a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
a = 2; b = 2; c = 2;
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
index 24b57f4353d95d..01076f76318997 100644
--- a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +3{{use of undeclared identifier 'c'}}
// expected-error at +2{{use of undeclared identifier 'b'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-checks.c b/clang/test/utils/update-verify-tests/Inputs/no-checks.c
index 8fd1f7cd333705..019698122c3f2b 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-checks.c
+++ b/clang/test/utils/update-verify-tests/Inputs/no-checks.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
bar = 2;
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
index e80548fbe50f2c..42c98d4f77b6ce 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'bar'}}
bar = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-diags.c b/clang/test/utils/update-verify-tests/Inputs/no-diags.c
index 66d169be439402..fddc2ad661fd2d 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-diags.c
+++ b/clang/test/utils/update-verify-tests/Inputs/no-diags.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{asdf}}
int a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
index 05230284945702..acb02badc5655a 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
@@ -1,4 +1,6 @@
// expected-no-diagnostics
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
int a = 2;
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
index 78b72e1357da76..e02b63dcd7a21b 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
+++ b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
// expected-no-diagnostics
void foo() {
a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
index d948ffce56189a..7fd85f2e064e28 100644
--- a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'a'}}
a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
index 3d63eaf0f1b878..b7a34e2dfa20d4 100644
--- a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
+++ b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify=check %s
+// RUN: diff %s %s.expected
void foo() {
a = 2; // check-error{{asdf}}
// expected-error at -1{ignored}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
index a877f86922123d..087a6be5ea4d2b 100644
--- a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify=check %s
+// RUN: diff %s %s.expected
void foo() {
a = 2; // check-error{{use of undeclared identifier 'a'}}
// expected-error at -1{ignored}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
index 5278ce0c57c319..e448800324e373 100644
--- a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
+++ b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
bar = 2; // expected-error {{asdf}}
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
index 8ba47f788319b1..ba4387a445fe08 100644
--- a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
bar = 2; // expected-error {{use of undeclared identifier 'bar'}}
}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
index 20b011bfc3d77e..e093a8c2ddddbb 100644
--- a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
+++ b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{asdf}}
bar = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
index e80548fbe50f2c..42c98d4f77b6ce 100644
--- a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
+++ b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
void foo() {
// expected-error at +1{{use of undeclared identifier 'bar'}}
bar = 2;
diff --git a/clang/test/utils/update-verify-tests/LitTests/.gitignore b/clang/test/utils/update-verify-tests/LitTests/.gitignore
new file mode 100644
index 00000000000000..307a6d636e7a28
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/LitTests/.gitignore
@@ -0,0 +1,4 @@
+*.c
+*.c.expected
+.lit_test_times.txt
+Output
diff --git a/clang/test/utils/update-verify-tests/LitTests/lit.cfg b/clang/test/utils/update-verify-tests/LitTests/lit.cfg
new file mode 100644
index 00000000000000..dbb22f13ef1303
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/LitTests/lit.cfg
@@ -0,0 +1,31 @@
+import lit.formats
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+from lit.llvm import llvm_config
+from os.path import dirname
+
+config.name = "Lit Update Verify Tests"
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.suffixes = [
+ ".c",
+]
+config.test_source_root = dirname(__file__)
+config.target_triple = None
+config.host_triple = None
+
+config.llvm_tools_dir = lit_config.path[0] # --path is explicitly passed by lit-plugin.test
+llvm_config.use_default_substitutions()
+llvm_config.use_clang()
+config.substitutions.append(("%clang_cc1", "%clang -cc1"))
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ config.clang_src_dir = dirname(dirname(dirname(dirname(config.test_source_root))))
+ utilspath = os.path.join(config.clang_src_dir, "utils")
+ sys.path.append(utilspath)
+ from UpdateVerifyTests.litplugin import verify_test_updater
+ # normally we'd append to the existing list, but when testing
+ # verify_test_updater we don't want diff_test_updater to accidentally interfere
+ lit_config.test_updaters = [verify_test_updater]
diff --git a/clang/test/utils/update-verify-tests/lit-plugin.test b/clang/test/utils/update-verify-tests/lit-plugin.test
new file mode 100644
index 00000000000000..f3bbf5ec36bfb5
--- /dev/null
+++ b/clang/test/utils/update-verify-tests/lit-plugin.test
@@ -0,0 +1,5 @@
+# RUN: cp %S/Inputs/*.c %S/LitTests
+# RUN: cp %S/Inputs/*.c.expected %S/LitTests
+# RUN: not %{lit} %S/LitTests
+# RUN: not %{lit} %S/LitTests --update-tests
+# RUN: %{lit} %S/LitTests
diff --git a/clang/test/utils/update-verify-tests/lit.local.cfg b/clang/test/utils/update-verify-tests/lit.local.cfg
index a0b6afccc25010..281c5be37a983a 100644
--- a/clang/test/utils/update-verify-tests/lit.local.cfg
+++ b/clang/test/utils/update-verify-tests/lit.local.cfg
@@ -12,6 +12,7 @@ if config.standalone_build:
config.unsupported = True
else:
config.suffixes = [".test"]
+ config.excludes.add("LitTests")
script_path = os.path.join(
config.clang_src_dir, "utils", "update-verify-tests.py"
@@ -23,3 +24,13 @@ else:
"%s %s" % (python, shell_quote(script_path)),
)
)
+ config.substitutions.append(("%{lit}", "%{lit-no-order-opt} --order=lexical"))
+
+ lit_path = os.path.join(config.llvm_tools_dir, "llvm-lit")
+ lit_path = os.path.abspath(lit_path)
+ config.substitutions.append(
+ (
+ "%{lit-no-order-opt}",
+ f"{python} {lit_path} -j1 --path {config.llvm_tools_dir}"
+ )
+)
diff --git a/clang/utils/UpdateVerifyTests/__init__.py b/clang/utils/UpdateVerifyTests/__init__.py
new file mode 100644
index 00000000000000..8b137891791fe9
--- /dev/null
+++ b/clang/utils/UpdateVerifyTests/__init__.py
@@ -0,0 +1 @@
+
diff --git a/clang/utils/UpdateVerifyTests/litplugin.py b/clang/utils/UpdateVerifyTests/litplugin.py
new file mode 100644
index 00000000000000..ee5fff19873f66
--- /dev/null
+++ b/clang/utils/UpdateVerifyTests/litplugin.py
@@ -0,0 +1,35 @@
+import sys
+from lit.formats import ShTest
+from UpdateVerifyTests.core import check_expectations
+import re
+
+verify_r = re.compile(r"-verify(?:=(\w+))?")
+
+def get_verify_prefixes(command):
+ def get_default(prefix):
+ if prefix:
+ return prefix
+ return "expected"
+
+ prefixes = set()
+ for arg in command.args:
+ m = verify_r.match(arg)
+ if not m:
+ continue
+ prefix = m[1]
+ if not prefix:
+ prefix = "expected"
+ prefixes.add(prefix)
+ return prefixes
+
+def verify_test_updater(result):
+ if not result.stderr:
+ return None
+ prefixes = get_verify_prefixes(result.command)
+ if not prefixes:
+ return None
+ if len(prefixes) > 1:
+ return f"update-verify-test: not updating because of multiple prefixes - {prefixes}"
+ [prefix] = prefixes
+ return check_expectations(result.stderr.splitlines(), prefix)
+
diff --git a/clang/utils/update-verify-tests.py b/clang/utils/update-verify-tests.py
index e2874a8c049ef3..1507a284b01c5a 100644
--- a/clang/utils/update-verify-tests.py
+++ b/clang/utils/update-verify-tests.py
@@ -21,6 +21,9 @@
Example usage:
clang -verify [file] | python3 update-verify-tests.py
clang -verify=check [file] | python3 update-verify-tests.py --prefix check
+
+This can also be invoked automatically by lit for failing '-verify' tests in Sema by running:
+ llvm-lit --update-tests clang/test/Sema
"""
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index 5dc712ae28370c..198a2bf3172330 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -38,6 +38,7 @@ def __init__(
parallelism_groups={},
per_test_coverage=False,
gtest_sharding=True,
+ update_tests=False,
):
# The name of the test runner.
self.progname = progname
@@ -89,6 +90,8 @@ def __init__(
self.parallelism_groups = parallelism_groups
self.per_test_coverage = per_test_coverage
self.gtest_sharding = bool(gtest_sharding)
+ self.update_tests = update_tests
+ self.test_updaters = []
@property
def maxIndividualTestTime(self):
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 19f35fc7e212f3..9b9612eaf7f3ef 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1166,6 +1166,18 @@ def executeScriptInternal(
str(result.timeoutReached),
)
+ if litConfig.update_tests:
+ for test_updater in litConfig.test_updaters:
+ try:
+ update_output = test_updater(result)
+ except Exception as e:
+ out += f"Exception occurred in test updater: {e}"
+ continue
+ if update_output:
+ for line in update_output.splitlines():
+ out += f"# {line}\n"
+ break
+
return out, err, exitCode, timeoutInfo
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index ed78256ee414b4..dcbe553c6d4827 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -204,6 +204,12 @@ def parse_args():
action="store_true",
help="Exit with status zero even if some tests fail",
)
+ execution_group.add_argument(
+ "--update-tests",
+ dest="update_tests",
+ action="store_true",
+ help="Try to update regression tests to reflect current behavior, if possible",
+ )
execution_test_time_group = execution_group.add_mutually_exclusive_group()
execution_test_time_group.add_argument(
"--skip-test-time-recording",
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index 8a6a0c2ed8089e..29eba49778e0ac 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -64,12 +64,17 @@ def __init__(self, lit_config, config):
self.with_environment("_TAG_REDIR_ERR", "TXT")
self.with_environment("_CEE_RUNOPTS", "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)")
+ if lit_config.update_tests:
+ self.use_lit_shell = True
+
# Choose between lit's internal shell pipeline runner and a real shell.
# If LIT_USE_INTERNAL_SHELL is in the environment, we use that as an
# override.
lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL")
if lit_shell_env:
self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
+ if not self.use_lit_shell and lit_config.update_tests:
+ print("note: --update-tests is not supported when using external shell")
if not self.use_lit_shell:
features.add("shell")
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 24ba804f0c363f..745e376de7d529 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -42,6 +42,7 @@ def main(builtin_params={}):
config_prefix=opts.configPrefix,
per_test_coverage=opts.per_test_coverage,
gtest_sharding=opts.gtest_sharding,
+ update_tests=opts.update_tests,
)
discovered_tests = lit.discovery.find_tests_for_inputs(
>From 5e0817746633736adc7e135160777c8ae39e0e45 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_olsson at apple.com>
Date: Thu, 29 Aug 2024 17:11:45 -0700
Subject: [PATCH 6/6] [Utils] Add UTC support for lit's --update-tests
Adds support for invoking the appropriate update_*_test_checks.py script
from lit. Checks the header comment for which script was used to
generate it in the first place, so only test cases that were already
generated are affected.
To support this the interface for test updater functions is expanded to
not only take a ShellCommandResult, but also the Test object. This makes
it easy to get the file path of the current test.
Also adds a --path flag to update_any_test_checks.py as a convenience
to avoid having to manually set the PATH variable.
---
clang/test/lit.cfg.py | 9 ++++
clang/utils/UpdateVerifyTests/litplugin.py | 4 +-
llvm/test/lit.cfg.py | 9 ++++
llvm/utils/lit/lit/TestRunner.py | 2 +-
llvm/utils/update_any_test_checks.py | 49 ++++++++++++++++++++--
5 files changed, 67 insertions(+), 6 deletions(-)
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index 92a3361ce672e2..c965d3167fe8eb 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -362,3 +362,12 @@ def calculate_arch_features(arch_string):
# possibly be present in system and user configuration files, so disable
# default configs for the test runs.
config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1"
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ utilspath = os.path.join(config.llvm_src_root, "utils")
+ sys.path.append(utilspath)
+ from update_any_test_checks import utc_lit_plugin
+ lit_config.test_updaters.append(utc_lit_plugin)
diff --git a/clang/utils/UpdateVerifyTests/litplugin.py b/clang/utils/UpdateVerifyTests/litplugin.py
index ee5fff19873f66..5e739e8e07dab6 100644
--- a/clang/utils/UpdateVerifyTests/litplugin.py
+++ b/clang/utils/UpdateVerifyTests/litplugin.py
@@ -21,8 +21,8 @@ def get_default(prefix):
prefix = "expected"
prefixes.add(prefix)
return prefixes
-
-def verify_test_updater(result):
+
+def verify_test_updater(result, test):
if not result.stderr:
return None
prefixes = get_verify_prefixes(result.command)
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 1e0dd0a7df34f1..b6859b32fdc743 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -622,3 +622,12 @@ def have_ld64_plugin_support():
if config.has_logf128:
config.available_features.add("has_logf128")
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ utilspath = os.path.join(config.llvm_src_root, "utils")
+ sys.path.append(utilspath)
+ from update_any_test_checks import utc_lit_plugin
+ lit_config.test_updaters.append(utc_lit_plugin)
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 9b9612eaf7f3ef..c0a1290f3c31f9 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1169,7 +1169,7 @@ def executeScriptInternal(
if litConfig.update_tests:
for test_updater in litConfig.test_updaters:
try:
- update_output = test_updater(result)
+ update_output = test_updater(result, test)
except Exception as e:
out += f"Exception occurred in test updater: {e}"
continue
diff --git a/llvm/utils/update_any_test_checks.py b/llvm/utils/update_any_test_checks.py
index e8eef1a46c504f..df1fc97778b019 100755
--- a/llvm/utils/update_any_test_checks.py
+++ b/llvm/utils/update_any_test_checks.py
@@ -34,9 +34,9 @@ def find_utc_tool(search_path, utc_name):
return None
-def run_utc_tool(utc_name, utc_tool, testname):
+def run_utc_tool(utc_name, utc_tool, testname, environment):
result = subprocess.run(
- [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+ [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
)
return (result.returncode, result.stdout, result.stderr)
@@ -60,6 +60,40 @@ def expand_listfile_args(arg_list):
return exp_arg_list
+def utc_lit_plugin(result, test):
+ testname = test.getFilePath()
+ if not testname:
+ return None
+
+ script_name = os.path.abspath(__file__)
+ utc_search_path = os.path.join(os.path.dirname(script_name), os.path.pardir)
+
+ with open(testname, "r") as f:
+ header = f.readline().strip()
+
+ m = RE_ASSERTIONS.search(header)
+ if m is None:
+ return None
+
+ utc_name = m.group(1)
+ utc_tool = find_utc_tool([utc_search_path], utc_name)
+ if not utc_tool:
+ return f"update-utc-tests: {utc_name} not found"
+
+ return_code, stdout, stderr = run_utc_tool(utc_name, utc_tool, testname, test.config.environment)
+
+ stderr = stderr.decode(errors="replace")
+ if return_code != 0:
+ if stderr:
+ return f"update-utc-tests: {utc_name} exited with return code {return_code}\n{stderr.rstrip()}"
+ return f"update-utc-tests: {utc_name} exited with return code {return_code}"
+
+ stdout = stdout.decode(errors="replace")
+ if stdout:
+ return f"update-utc-tests: updated {testname}\n{stdout.rstrip()}"
+ return f"update-utc-tests: updated {testname}"
+
+
def main():
from argparse import RawTextHelpFormatter
@@ -78,6 +112,11 @@ def main():
nargs="*",
help="Additional directories to scan for update_*_test_checks scripts",
)
+ parser.add_argument(
+ "--path",
+ help="""Additional directories to scan for executables invoked by the update_*_test_checks scripts,
+separated by the platform path separator""",
+ )
parser.add_argument("tests", nargs="+")
config = parser.parse_args()
@@ -88,6 +127,10 @@ def main():
script_name = os.path.abspath(__file__)
utc_search_path.append(os.path.join(os.path.dirname(script_name), os.path.pardir))
+ local_env = os.environ.copy()
+ if config.path:
+ local_env["PATH"] = config.path + os.pathsep + local_env["PATH"]
+
not_autogenerated = []
utc_tools = {}
have_error = False
@@ -117,7 +160,7 @@ def main():
continue
future = executor.submit(
- run_utc_tool, utc_name, utc_tools[utc_name], testname
+ run_utc_tool, utc_name, utc_tools[utc_name], testname, local_env
)
jobs.append((testname, future))
More information about the cfe-commits
mailing list