[clang] [UVT] add update-verify-tests.py (PR #97369)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Jul 1 18:32:03 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang
Author: Henrik G. Olsson (hnrklssn)
<details>
<summary>Changes</summary>
Adds a python script to automatically take output from a failed clang -verify test and update the test case(s) to expect the new behaviour.
---
Full diff: https://github.com/llvm/llvm-project/pull/97369.diff
1 Files Affected:
- (added) clang/utils/update-verify-tests.py (+321)
``````````diff
diff --git a/clang/utils/update-verify-tests.py b/clang/utils/update-verify-tests.py
new file mode 100644
index 0000000000000..95eca3af61a72
--- /dev/null
+++ b/clang/utils/update-verify-tests.py
@@ -0,0 +1,321 @@
+import sys
+import re
+
+"""
+ Pipe output from clang's -verify into this script to have the test case updated to expect the actual diagnostic output.
+ When inserting new expected-* checks it will place them on the line before the location of the diagnostic, with an @+1,
+ or @+N for some N if there are multiple diagnostics emitted on the same line. If the current checks are using @-N for
+ this line, the new check will follow that convention also.
+ Existing checks will be left untouched as much as possible, including their location and whitespace content, to minimize
+ diffs. If inaccurate their count will be updated, or the check removed entirely.
+
+ Missing features:
+ - custom prefix support (-verify=my-prefix)
+ - multiple prefixes on the same line (-verify=my-prefix,my-other-prefix)
+ - multiple prefixes on separate RUN lines (RUN: -verify=my-prefix\nRUN: -verify my-other-prefix)
+ - regexes with expected-*-re: existing ones will be left untouched if accurate, but the script will abort if there are any
+ diagnostic mismatches on the same line.
+ - multiple checks targeting the same line are supported, but a line may only contain one check
+ - if multiple checks targeting the same line are failing the script is not guaranteed to produce a minimal diff
+
+Example usage:
+ build/bin/llvm-lit clang/test/Sema/ --no-progress-bar -v | python3 update-verify-tests.py
+"""
+
+class KnownException(Exception):
+ pass
+
+def parse_error_category(s):
+ parts = s.split("diagnostics")
+ diag_category = parts[0]
+ category_parts = parts[0].strip().strip("'").split("-")
+ expected = category_parts[0]
+ if expected != "expected":
+ raise Exception(f"expected 'expected', but found '{expected}'. Custom verify prefixes are not supported.")
+ diag_category = category_parts[1]
+ if "seen but not expected" in parts[1]:
+ seen = True
+ elif "expected but not seen" in parts[1]:
+ seen = False
+ else:
+ raise KnownException(f"unexpected category '{parts[1]}'")
+ return (diag_category, seen)
+
+diag_error_re = re.compile(r"File (\S+) Line (\d+): (.+)")
+diag_error_re2 = re.compile(r"File \S+ Line \d+ \(directive at (\S+):(\d+)\): (.+)")
+def parse_diag_error(s):
+ m = diag_error_re2.match(s)
+ if not m:
+ m = diag_error_re.match(s)
+ if not m:
+ return None
+ return (m.group(1), int(m.group(2)), m.group(3))
+
+class Line:
+ def __init__(self, content, line_n):
+ self.content = content
+ self.diag = None
+ self.line_n = line_n
+ self.related_diags = []
+ self.targeting_diags = []
+ def update_line_n(self, n):
+ if self.diag and not self.diag.line_is_absolute:
+ self.diag.orig_target_line_n += n - self.line_n
+ self.line_n = n
+ for diag in self.targeting_diags:
+ if diag.line_is_absolute:
+ diag.orig_target_line_n = n
+ else:
+ diag.orig_target_line_n = n - diag.line.line_n
+ for diag in self.related_diags:
+ if not diag.line_is_absolute:
+ pass
+ def render(self):
+ if not self.diag:
+ return self.content
+ assert("{{DIAG}}" in self.content)
+ res = self.content.replace("{{DIAG}}", self.diag.render())
+ if not res.strip():
+ return ""
+ return res
+
+class Diag:
+ def __init__(self, diag_content, category, targeted_line_n, line_is_absolute, count, line, is_re, whitespace_strings):
+ self.diag_content = diag_content
+ self.category = category
+ self.orig_target_line_n = targeted_line_n
+ self.line_is_absolute = line_is_absolute
+ self.count = count
+ self.line = line
+ self.target = None
+ self.is_re = is_re
+ self.absolute_target()
+ self.whitespace_strings = whitespace_strings
+
+ def add(self):
+ if targeted_line > 0:
+ targeted_line += 1
+ elif targeted_line < 0:
+ targeted_line -= 1
+
+ def absolute_target(self):
+ if self.line_is_absolute:
+ res = self.orig_target_line_n
+ else:
+ res = self.line.line_n + self.orig_target_line_n
+ if self.target:
+ assert(self.line.line_n == res)
+ return res
+
+ def relative_target(self):
+ return self.absolute_target() - self.line.line_n
+
+ def render(self):
+ assert(self.count >= 0)
+ if self.count == 0:
+ return ""
+ line_location_s = ""
+ if self.relative_target() != 0:
+ if self.line_is_absolute:
+ line_location_s = f"@{self.absolute_target()}"
+ elif self.relative_target() > 0:
+ line_location_s = f"@+{self.relative_target()}"
+ else:
+ line_location_s = f"@{self.relative_target()}" # the minus sign is implicit
+ count_s = "" if self.count == 1 else f"{self.count}"
+ re_s = "-re" if self.is_re else ""
+ if self.whitespace_strings:
+ whitespace1_s = self.whitespace_strings[0]
+ whitespace2_s = self.whitespace_strings[1]
+ whitespace3_s = self.whitespace_strings[2]
+ whitespace4_s = self.whitespace_strings[3]
+ else:
+ whitespace1_s = " "
+ whitespace2_s = ""
+ whitespace3_s = ""
+ whitespace4_s = ""
+ if count_s and not whitespace3_s:
+ whitespace3_s = " "
+ return f"//{whitespace1_s}expected-{self.category}{re_s}{whitespace2_s}{line_location_s}{whitespace3_s}{count_s}{whitespace4_s}{{{{{self.diag_content}}}}}"
+
+expected_diag_re = re.compile(r"//(\s*)expected-(note|warning|error)(-re)?(\s*)(@[+-]?\d+)?(\s*)(\d+)?(\s*)\{\{(.*)\}\}")
+def parse_diag(line, filename, lines):
+ s = line.content
+ ms = expected_diag_re.findall(s)
+ if not ms:
+ return None
+ if len(ms) > 1:
+ print(f"multiple diags on line {filename}:{line.line_n}. Aborting due to missing implementation.")
+ sys.exit(1)
+ [whitespace1_s, category_s, re_s, whitespace2_s, target_line_s, whitespace3_s, count_s, whitespace4_s, diag_s] = ms[0]
+ if not target_line_s:
+ target_line_n = 0
+ is_absolute = False
+ elif target_line_s.startswith("@+"):
+ target_line_n = int(target_line_s[2:])
+ is_absolute = False
+ elif target_line_s.startswith("@-"):
+ target_line_n = int(target_line_s[1:])
+ is_absolute = False
+ else:
+ target_line_n = int(target_line_s[1:])
+ is_absolute = True
+ count = int(count_s) if count_s else 1
+ line.content = expected_diag_re.sub("{{DIAG}}", s)
+
+ return Diag(diag_s, category_s, target_line_n, is_absolute, count, line, bool(re_s), [whitespace1_s, whitespace2_s, whitespace3_s, whitespace4_s])
+
+def link_line_diags(lines, diag):
+ line_n = diag.line.line_n
+ target_line_n = diag.absolute_target()
+ step = 1 if target_line_n < line_n else -1
+ for i in range(target_line_n, line_n, step):
+ lines[i-1].related_diags.append(diag)
+
+def add_line(new_line, lines):
+ lines.insert(new_line.line_n-1, new_line)
+ for i in range(new_line.line_n, len(lines)):
+ line = lines[i]
+ assert(line.line_n == i)
+ line.update_line_n(i+1)
+ assert(all(line.line_n == i+1 for i, line in enumerate(lines)))
+
+indent_re = re.compile(r"\s*")
+def get_indent(s):
+ return indent_re.match(s).group(0)
+
+def add_diag(line_n, diag_s, diag_category, lines):
+ target = lines[line_n - 1]
+ for other in target.targeting_diags:
+ if other.is_re:
+ raise KnownException("mismatching diag on line with regex matcher. Skipping due to missing implementation")
+ reverse = True if [other for other in target.targeting_diags if other.relative_target() < 0] else False
+
+ targeting = [other for other in target.targeting_diags if not other.line_is_absolute]
+ targeting.sort(reverse=reverse, key=lambda d: d.relative_target())
+ prev_offset = 0
+ prev_line = target
+ direction = -1 if reverse else 1
+ for d in targeting:
+ if d.relative_target() != prev_offset + direction:
+ break
+ prev_offset = d.relative_target()
+ prev_line = d.line
+ total_offset = prev_offset - 1 if reverse else prev_offset + 1
+ if reverse:
+ new_line_n = prev_line.line_n + 1
+ else:
+ new_line_n = prev_line.line_n
+ assert(new_line_n == line_n + (not reverse) - total_offset)
+
+ new_line = Line(get_indent(prev_line.content) + "{{DIAG}}\n", new_line_n)
+ new_line.related_diags = list(prev_line.related_diags)
+ add_line(new_line, lines)
+
+ new_diag = Diag(diag_s, diag_category, total_offset, False, 1, new_line, False, None)
+ new_line.diag = new_diag
+ new_diag.target_line = target
+ assert(type(new_diag) != str)
+ target.targeting_diags.append(new_diag)
+ link_line_diags(lines, new_diag)
+
+updated_test_files = set()
+def update_test_file(filename, diag_errors):
+ print(f"updating test file {filename}")
+ if filename in updated_test_files:
+ print(f"{filename} already updated, but got new output - expect incorrect results")
+ else:
+ updated_test_files.add(filename)
+ with open(filename, 'r') as f:
+ lines = [Line(line, i+1) for i, line in enumerate(f.readlines())]
+ for line in lines:
+ diag = parse_diag(line, filename, lines)
+ if diag:
+ line.diag = diag
+ diag.target_line = lines[diag.absolute_target() - 1]
+ link_line_diags(lines, diag)
+ lines[diag.absolute_target() - 1].targeting_diags.append(diag)
+
+ for (line_n, diag_s, diag_category, seen) in diag_errors:
+ if seen:
+ continue
+ # this is a diagnostic expected but not seen
+ assert(lines[line_n - 1].diag)
+ if diag_s != lines[line_n - 1].diag.diag_content:
+ raise KnownException(f"{filename}:{line_n} - found diag {lines[line_n - 1].diag.diag_content} but expected {diag_s}")
+ if diag_category != lines[line_n - 1].diag.category:
+ raise KnownException(f"{filename}:{line_n} - found {lines[line_n - 1].diag.category} diag but expected {diag_category}")
+ lines[line_n - 1].diag.count -= 1
+ diag_errors_left = []
+ diag_errors.sort(reverse=True, key=lambda t: t[0])
+ for (line_n, diag_s, diag_category, seen) in diag_errors:
+ if not seen:
+ continue
+ target = lines[line_n - 1]
+ other_diags = [d for d in target.targeting_diags if d.diag_content == diag_s and d.category == diag_category]
+ other_diag = other_diags[0] if other_diags else None
+ if other_diag:
+ other_diag.count += 1
+ else:
+ diag_errors_left.append((line_n, diag_s, diag_category))
+ for (line_n, diag_s, diag_category) in diag_errors_left:
+ add_diag(line_n, diag_s, diag_category, lines)
+ with open(filename, 'w') as f:
+ for line in lines:
+ f.write(line.render())
+
+def update_test_files(errors):
+ errors_by_file = {}
+ for ((filename, line, diag_s), (diag_category, seen)) in errors:
+ if filename not in errors_by_file:
+ errors_by_file[filename] = []
+ errors_by_file[filename].append((line, diag_s, diag_category, seen))
+ for filename, diag_errors in errors_by_file.items():
+ try:
+ update_test_file(filename, diag_errors)
+ except KnownException as e:
+ print(f"{filename} - ERROR: {e}")
+ print("continuing...")
+curr = []
+curr_category = None
+curr_run_line = None
+lines_since_run = []
+for line in sys.stdin.readlines():
+ lines_since_run.append(line)
+ try:
+ if line.startswith("RUN:"):
+ if curr:
+ update_test_files(curr)
+ curr = []
+ lines_since_run = [line]
+ curr_run_line = line
+ else:
+ for line in lines_since_run:
+ print(line, end="")
+ print("====================")
+ print("no mismatching diagnostics found since last RUN line")
+ continue
+ if line.startswith("error: "):
+ if "no expected directives found" in line:
+ print(f"no expected directives found for RUN line '{curr_run_line.strip()}'. Add 'expected-no-diagnostics' manually if this is intended.")
+ continue
+ curr_category = parse_error_category(line[len("error: "):])
+ continue
+
+ diag_error = parse_diag_error(line.strip())
+ if diag_error:
+ curr.append((diag_error, curr_category))
+ except Exception as e:
+ for line in lines_since_run:
+ print(line, end="")
+ print("====================")
+ print(e)
+ sys.exit(1)
+if curr:
+ update_test_files(curr)
+ print("done!")
+else:
+ for line in lines_since_run:
+ print(line, end="")
+ print("====================")
+ print("no mismatching diagnostics found")
``````````
</details>
https://github.com/llvm/llvm-project/pull/97369
More information about the cfe-commits
mailing list