[llvm] [UpdateTestChecks] Don't fail silently when conflicting CHECK lines means no checks are generated for some functions (PR #159321)
Alex Bradbury via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 23 08:31:30 PDT 2025
https://github.com/asb updated https://github.com/llvm/llvm-project/pull/159321
>From a3bd517e3540289ca3ffa7a7a512191b63900628 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb at igalia.com>
Date: Wed, 17 Sep 2025 11:58:39 +0100
Subject: [PATCH 1/7] [UpdateTestChecks] Don't fail silently when conflicting
CHECK lines means no checks are generated for some functions
There is a warning that triggers if you (for instance) run
`update_llc_test_checks.py` on an input where _all_ functions have
conflicting check lines and so no checks are generated. However, there
are no warnings emitted at all for the case where some functions have
non-conflicting check lines but others don't. This is a source of
frustration because running update_llc_test_checks can result in all
check lines being removed when such a conflict exists with no warning,
meaning we have to be extra vigilant inspecting the diff. I've also
personally wasted time tracking down the source of the dropped lines
assuming that update_test_checks would emit a warning in such cases.
This change adds logic to emit warnings on a function-by-function basis
for any RUN that has conflicting prefixes meaning no output is
generated. This subsumes the previous warning for when _all_ functions
conflict.
---
.../Inputs/conflicting-prefixes.ll | 16 +++++
.../Inputs/conflicting-prefixes.ll.expected | 21 ++++++
.../conflicting-prefixes.test | 7 ++
.../prefix-never-matches.test | 2 +-
llvm/utils/UpdateTestChecks/common.py | 65 +++++++++++++------
5 files changed, 90 insertions(+), 21 deletions(-)
create mode 100644 llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll
create mode 100644 llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll.expected
create mode 100644 llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll
new file mode 100644
index 0000000000000..fdc53951d6bb0
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll
@@ -0,0 +1,16 @@
+; RUN: sed 's/RETVAL/1/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/RETVAL/2/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/RETVAL/3/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKB %s
+; RUN: sed 's/RETVAL/4/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKB %s
+
+define i32 @foo() {
+ ret i32 RETVAL
+}
+
+define i32 @bar() {
+ ret i32 100
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll.expected
new file mode 100644
index 0000000000000..b3cad11e2ec1d
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll.expected
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no-generate-body-for-unused-prefixes
+; RUN: sed 's/RETVAL/1/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/RETVAL/2/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/RETVAL/3/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKB %s
+; RUN: sed 's/RETVAL/4/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKB %s
+
+define i32 @foo() {
+ ret i32 RETVAL
+}
+
+define i32 @bar() {
+; CHECK-LABEL: bar:
+; CHECK: # %bb.0:
+; CHECK-NEXT: li a0, 100
+; CHECK-NEXT: ret
+ ret i32 100
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test
new file mode 100644
index 0000000000000..c9a6a7268d0b3
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test
@@ -0,0 +1,7 @@
+# REQUIRES: riscv-registered-target
+
+# RUN: cp -f %S/Inputs/conflicting-prefixes.ll %t.ll
+# RUN: %update_llc_test_checks --no-generate-body-for-unused-prefixes %t.ll 2>&1 | FileCheck %s
+# RUN: diff -u %S/Inputs/conflicting-prefixes.ll.expected %t.ll
+
+# CHECK: WARNING: For function 'foo', the following RUN lines will not generate checks due to conflicting output
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/prefix-never-matches.test b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/prefix-never-matches.test
index 2e75148addd84..90ae70bda64d9 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/prefix-never-matches.test
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/prefix-never-matches.test
@@ -4,5 +4,5 @@
# RUN: %update_llc_test_checks --no-generate-body-for-unused-prefixes %t.ll 2>&1 | FileCheck %s
# RUN: FileCheck --input-file=%t.ll %s --check-prefix=OUTPUT
-# CHECK: WARNING: Prefix A had conflicting output
+# CHECK: WARNING: For function 'fold_v2i64', the following RUN lines will not generate checks due to conflicting output
# OUTPUT-NOT: A:
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 1c795afa9e700..c1cb195852a79 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -882,6 +882,7 @@ def __str__(self):
class FunctionTestBuilder:
def __init__(self, run_list, flags, scrubber_args, path, ginfo):
+ self._run_list = run_list
self._verbose = flags.verbose
self._record_args = flags.function_signature
self._check_attributes = flags.check_attributes
@@ -918,14 +919,52 @@ def __init__(self, run_list, flags, scrubber_args, path, ginfo):
self._global_var_dict.update({prefix: dict()})
def finish_and_get_func_dict(self):
- for prefix in self.get_failed_prefixes():
- warn(
- "Prefix %s had conflicting output from different RUN lines for all functions in test %s"
- % (
- prefix,
- self._path,
+ all_funcs = set()
+ for prefix in self._func_dict:
+ all_funcs.update(self._func_dict[prefix].keys())
+
+ warnings_to_print = collections.defaultdict(list)
+ for func in sorted(list(all_funcs)):
+ for i, run_info in enumerate(self._run_list):
+ prefixes = run_info[0]
+ if not prefixes:
+ continue
+
+ # Check if this RUN line produces this function at all.
+ run_contains_func = True
+ for p in prefixes:
+ if func not in self._func_dict.get(p, {}):
+ run_contains_func = False
+ break
+ if not run_contains_func:
+ continue
+
+ # Check if this RUN line can print any checks for this
+ # function. It can't if all of its prefixes have conflicting
+ # (None) output.
+ can_print_for_this_run = False
+ for p in prefixes:
+ if self._func_dict[p].get(func) is not None:
+ can_print_for_this_run = True
+ break
+
+ if not can_print_for_this_run:
+ warnings_to_print[func].append((i, prefixes))
+
+ for func, warning_info in warnings_to_print.items():
+ conflict_strs = []
+ for run_index, prefixes in warning_info:
+ conflict_strs.append(
+ "RUN #{} (prefixes: {})".format(
+ run_index + 1, ", ".join(prefixes)
+ )
)
+ warn(
+ "For function '{}', the following RUN lines will not generate checks due to conflicting output: {}".format(
+ func, ", ".join(conflict_strs)),
+ test_file=self._path,
)
+
return self._func_dict
def func_order(self):
@@ -1078,20 +1117,6 @@ def processed_prefixes(self, prefixes):
"""
self._processed_prefixes.update(prefixes)
- def get_failed_prefixes(self):
- # This returns the list of those prefixes that failed to match any function,
- # because there were conflicting bodies produced by different RUN lines, in
- # all instances of the prefix.
- for prefix in self._func_dict:
- if self._func_dict[prefix] and (
- not [
- fct
- for fct in self._func_dict[prefix]
- if self._func_dict[prefix][fct] is not None
- ]
- ):
- yield prefix
-
##### Generator of LLVM IR CHECK lines
>From 3d8c2a817e18f0500e324aa0e38b4266700c8d33 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb at igalia.com>
Date: Wed, 17 Sep 2025 12:12:50 +0100
Subject: [PATCH 2/7] Apply reformatting suggested by darker
---
llvm/utils/UpdateTestChecks/common.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index c1cb195852a79..a5fa97217330b 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -955,13 +955,12 @@ def finish_and_get_func_dict(self):
conflict_strs = []
for run_index, prefixes in warning_info:
conflict_strs.append(
- "RUN #{} (prefixes: {})".format(
- run_index + 1, ", ".join(prefixes)
- )
+ "RUN #{} (prefixes: {})".format(run_index + 1, ", ".join(prefixes))
)
warn(
"For function '{}', the following RUN lines will not generate checks due to conflicting output: {}".format(
- func, ", ".join(conflict_strs)),
+ func, ", ".join(conflict_strs)
+ ),
test_file=self._path,
)
>From be93e6ecc1494c367ec7afc3a710e53336c6cbe3 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb at igalia.com>
Date: Mon, 22 Sep 2025 11:36:43 +0100
Subject: [PATCH 3/7] Address review comments
---
llvm/utils/UpdateTestChecks/common.py | 29 ++++++++++++---------------
1 file changed, 13 insertions(+), 16 deletions(-)
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index a5fa97217330b..f0f69b9ddb66d 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -918,6 +918,13 @@ def __init__(self, run_list, flags, scrubber_args, path, ginfo):
self._func_order.update({prefix: []})
self._global_var_dict.update({prefix: dict()})
+ # Return true if there is conflicting output for different runs for the
+ # given prefix and function name.
+ def has_conflicting_output(self, prefix, func):
+ # There was conflicting output if the func_dict is None for this
+ # prefix and function.
+ return self._func_dict[prefix].get(func) is None
+
def finish_and_get_func_dict(self):
all_funcs = set()
for prefix in self._func_dict:
@@ -930,24 +937,16 @@ def finish_and_get_func_dict(self):
if not prefixes:
continue
- # Check if this RUN line produces this function at all.
- run_contains_func = True
- for p in prefixes:
- if func not in self._func_dict.get(p, {}):
- run_contains_func = False
- break
+ # Check if this RUN line produces this function at all. If
+ # not, we can skip analysing this function for this RUN.
+ run_contains_func = all(func in self._func_dict.get(p, {}) for p in prefixes)
if not run_contains_func:
continue
# Check if this RUN line can print any checks for this
# function. It can't if all of its prefixes have conflicting
# (None) output.
- can_print_for_this_run = False
- for p in prefixes:
- if self._func_dict[p].get(func) is not None:
- can_print_for_this_run = True
- break
-
+ can_print_for_this_run = not all(self.has_conflicting_output(p, func) for p in prefixes)
if not can_print_for_this_run:
warnings_to_print[func].append((i, prefixes))
@@ -955,12 +954,10 @@ def finish_and_get_func_dict(self):
conflict_strs = []
for run_index, prefixes in warning_info:
conflict_strs.append(
- "RUN #{} (prefixes: {})".format(run_index + 1, ", ".join(prefixes))
+ f"RUN #{run_index + 1} (prefixes: {', '.join(prefixes)})"
)
warn(
- "For function '{}', the following RUN lines will not generate checks due to conflicting output: {}".format(
- func, ", ".join(conflict_strs)
- ),
+ f"For function '{func}', the following RUN lines will not generate checks due to conflicting output: {', '.join(conflict_strs)}",
test_file=self._path,
)
>From c3cc1ba2c059de313dcafa9419fc114b603cbd19 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb at igalia.com>
Date: Tue, 23 Sep 2025 10:24:36 +0100
Subject: [PATCH 4/7] Check full warning in conflicting-prefixes.test
---
.../update_llc_test_checks/conflicting-prefixes.test | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test
index c9a6a7268d0b3..e835b5f83f9a0 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test
@@ -4,4 +4,4 @@
# RUN: %update_llc_test_checks --no-generate-body-for-unused-prefixes %t.ll 2>&1 | FileCheck %s
# RUN: diff -u %S/Inputs/conflicting-prefixes.ll.expected %t.ll
-# CHECK: WARNING: For function 'foo', the following RUN lines will not generate checks due to conflicting output
+# CHECK: WARNING: For function 'foo', the following RUN lines will not generate checks due to conflicting output: RUN #1 (prefixes: CHECK, CHECKA), RUN #2 (prefixes: CHECK, CHECKA), RUN #3 (prefixes: CHECK, CHECKB), RUN #4 (prefixes: CHECK, CHECKB):
>From 4ca0f2225b2a6a31131bd8f53f71b4aa0d3572fb Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb at igalia.com>
Date: Tue, 23 Sep 2025 10:25:19 +0100
Subject: [PATCH 5/7] Avoid double not
---
llvm/utils/UpdateTestChecks/common.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index f0f69b9ddb66d..fb1ea75117615 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -946,8 +946,8 @@ def finish_and_get_func_dict(self):
# Check if this RUN line can print any checks for this
# function. It can't if all of its prefixes have conflicting
# (None) output.
- can_print_for_this_run = not all(self.has_conflicting_output(p, func) for p in prefixes)
- if not can_print_for_this_run:
+ cannot_print_for_this_run = all(self.has_conflicting_output(p, func) for p in prefixes)
+ if cannot_print_for_this_run:
warnings_to_print[func].append((i, prefixes))
for func, warning_info in warnings_to_print.items():
>From e8ffdc9c997c2e3e2984a786db06155a20345d91 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb at igalia.com>
Date: Tue, 23 Sep 2025 16:21:17 +0100
Subject: [PATCH 6/7] Add test case that checks we don't print false positive
warning when some functions aren't present for some RUN lines
---
.../Inputs/differing-set-of-functions.ll | 14 +++++
.../differing-set-of-functions.ll.expected | 54 +++++++++++++++++++
.../differing-set-of-functions.test | 11 ++++
3 files changed, 79 insertions(+)
create mode 100644 llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll
create mode 100644 llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll.expected
create mode 100644 llvm/test/tools/UpdateTestChecks/update_llc_test_checks/differing-set-of-functions.test
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll
new file mode 100644
index 0000000000000..6c3c66e1a7229
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll
@@ -0,0 +1,14 @@
+; RUN: sed 's/FN/foo/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/FN/foo/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKB %s
+; RUN: sed 's/FN/bar/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKC %s
+
+define i32 @FN() {
+ ret i32 1
+}
+
+define i32 @common() {
+ ret i32 100
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll.expected
new file mode 100644
index 0000000000000..b851f3a3ae249
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll.expected
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --include-generated-funcs
+; RUN: sed 's/FN/foo/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/FN/foo/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKB %s
+; RUN: sed 's/FN/bar/g' %s | llc -mtriple=riscv32 \
+; RUN: | FileCheck -check-prefixes=CHECK,CHECKC %s
+
+define i32 @FN() {
+ ret i32 1
+}
+
+define i32 @common() {
+ ret i32 100
+}
+; CHECK-LABEL: foo:
+; CHECK: # %bb.0:
+; CHECK-NEXT: li a0, 1
+; CHECK-NEXT: ret
+;
+; CHECK-LABEL: common:
+; CHECK: # %bb.0:
+; CHECK-NEXT: li a0, 100
+; CHECK-NEXT: ret
+;
+; CHECKA-LABEL: foo:
+; CHECKA: # %bb.0:
+; CHECKA-NEXT: li a0, 1
+; CHECKA-NEXT: ret
+;
+; CHECKA-LABEL: common:
+; CHECKA: # %bb.0:
+; CHECKA-NEXT: li a0, 100
+; CHECKA-NEXT: ret
+;
+; CHECKB-LABEL: foo:
+; CHECKB: # %bb.0:
+; CHECKB-NEXT: li a0, 1
+; CHECKB-NEXT: ret
+;
+; CHECKB-LABEL: common:
+; CHECKB: # %bb.0:
+; CHECKB-NEXT: li a0, 100
+; CHECKB-NEXT: ret
+;
+; CHECKC-LABEL: bar:
+; CHECKC: # %bb.0:
+; CHECKC-NEXT: li a0, 1
+; CHECKC-NEXT: ret
+;
+; CHECKC-LABEL: common:
+; CHECKC: # %bb.0:
+; CHECKC-NEXT: li a0, 100
+; CHECKC-NEXT: ret
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/differing-set-of-functions.test b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/differing-set-of-functions.test
new file mode 100644
index 0000000000000..749f3f2a528c1
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/differing-set-of-functions.test
@@ -0,0 +1,11 @@
+# REQUIRES: riscv-registered-target
+
+# RUN: cp -f %S/Inputs/differing-set-of-functions.ll %t.ll
+# RUN: %update_llc_test_checks --include-generated-funcs %t.ll 2>&1 | FileCheck --allow-empty %s
+# RUN: diff -u %S/Inputs/differing-set-of-functions.ll.expected %t.ll
+
+# We shouldn't print the warning for clashing CHECK prefixes in the case that
+# we're trying to handle a function that is only present for some RUN lines.
+# Better warning behaviour than this might be possible.
+
+# CHECK-NOT: WARNING
>From 7e47a56e3dab83d3fc0b39399c696ec4dd1a5714 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb at igalia.com>
Date: Tue, 23 Sep 2025 16:22:56 +0100
Subject: [PATCH 7/7] Attempt to fix darker code formatting issues
---
llvm/utils/UpdateTestChecks/common.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index fb1ea75117615..a5e3c39bfdecd 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -939,14 +939,18 @@ def finish_and_get_func_dict(self):
# Check if this RUN line produces this function at all. If
# not, we can skip analysing this function for this RUN.
- run_contains_func = all(func in self._func_dict.get(p, {}) for p in prefixes)
+ run_contains_func = all(
+ func in self._func_dict.get(p, {}) for p in prefixes
+ )
if not run_contains_func:
continue
# Check if this RUN line can print any checks for this
# function. It can't if all of its prefixes have conflicting
# (None) output.
- cannot_print_for_this_run = all(self.has_conflicting_output(p, func) for p in prefixes)
+ cannot_print_for_this_run = all(
+ self.has_conflicting_output(p, func) for p in prefixes
+ )
if cannot_print_for_this_run:
warnings_to_print[func].append((i, prefixes))
@@ -954,7 +958,7 @@ def finish_and_get_func_dict(self):
conflict_strs = []
for run_index, prefixes in warning_info:
conflict_strs.append(
- f"RUN #{run_index + 1} (prefixes: {', '.join(prefixes)})"
+ f"RUN #{run_index + 1} (prefixes: {', '.join(prefixes)})"
)
warn(
f"For function '{func}', the following RUN lines will not generate checks due to conflicting output: {', '.join(conflict_strs)}",
More information about the llvm-commits
mailing list