[llvm] [lit] introduce --show-attempts-count (PR #142413)
Konrad Kleine via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 2 09:16:59 PDT 2025
https://github.com/kwk updated https://github.com/llvm/llvm-project/pull/142413
>From ccffc2eccf52664b6eca8b8d6ab68f8b8a65de21 Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Mon, 2 Jun 2025 11:48:35 +0000
Subject: [PATCH 1/3] [lit] introduce --show-attempts-count
This introduces `--show-attempts-count` output option to `lit`.
It shows the number of attempts needed for a test to pass. It also shows
the maximum number of attempts that were allowed for this test. This option
is useful when you want to understand how many attempts took for a flaky
test (``FLAKYPASS``) to pass.
This is how the test output looks like *without* `--show-attempts-count`:
```
PASS: your-test-suite :: your-first-test.py (1 of 3)
FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2)
```
This is the output *with* `--show-attempts-count`:
```
PASS: your-test-suite :: your-first-test.py (1 of 2)
FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2) [attempts=3,max_allowed_attempts=4]
```
In this case `your-second-test.py` was executed three times and it
succeeded after the third time (`attempts=3`). Technically another run would have
been possible (`max_allowed_attempts=4`).
We will only append the extra information when a test was allowed more
than one attempt to succeed (i.e. see :option:`--max-retries-per-test`).
NOTE: Additionally this is a fixup for #141851 where the tests were not
quite right. `max-retries-per-test/allow-retries-test_retry_attempts/test.py`
was added but never used there. Now we're calling it. To correlate better between
the test output and the test script I've used higher numbers of max
allowed retries.
---
llvm/docs/CommandGuide/lit.rst | 28 +++++++++++++++++++
llvm/utils/lit/lit/Test.py | 8 +++++-
llvm/utils/lit/lit/TestRunner.py | 4 ++-
llvm/utils/lit/lit/cl_arguments.py | 6 ++++
llvm/utils/lit/lit/display.py | 27 ++++++++++++++++--
.../test.py | 2 +-
.../allow-retries-test_retry_attempts/test.py | 2 +-
.../lit.cfg | 2 +-
llvm/utils/lit/tests/allow-retries.py | 22 ++++++++++-----
9 files changed, 87 insertions(+), 14 deletions(-)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 86879f870e06e..73219cb7bfe98 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -144,6 +144,34 @@ OUTPUT OPTIONS
Show expectedly failed tests.
+.. option:: --show-attempts-count
+
+ Show the number of attempts needed for a test to pass. It also shows the
+ maximum number of attempts that were allowed for this test. This option is
+ useful when you want to understand how many attempts took for a flaky test
+ (``FLAKYPASS``) to pass.
+
+ This is how the test output looks like *without* :option:`--show-attempts-count`
+
+ .. code-block:: none
+
+ PASS: your-test-suite :: your-first-test.py (1 of 3)
+ FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2)
+
+ This is the output *with* :option:`--show-attempts-count`
+
+ .. code-block:: none
+
+ PASS: your-test-suite :: your-first-test.py (1 of 2)
+ FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2) [attempts=3,max_allowed_attempts=4]
+
+ In this case ``your-second-test.py`` was executed three times and it succeeded
+ after the third time (``attempts=3``). Technically another run would have been
+ possible (``max_allowed_attempts=4``).
+
+ We will only append the extra information when a test was allowed more than one
+ attempt to succeed (i.e. see :option:`--max-retries-per-test`).
+
.. _execution-options:
EXECUTION OPTIONS
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 051062706f856..fd8fc4c5f89ce 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -151,7 +151,9 @@ def toMetricValue(value):
class Result(object):
"""Wrapper for the results of executing an individual test."""
- def __init__(self, code, output="", elapsed=None):
+ def __init__(
+ self, code, output="", elapsed=None, attempts=None, max_allowed_attempts=None
+ ):
# The result code.
self.code = code
# The test output.
@@ -164,6 +166,10 @@ def __init__(self, code, output="", elapsed=None):
self.metrics = {}
# The micro-test results reported by this test.
self.microResults = {}
+ # How often was the test run?
+ self.attempts = attempts
+ # How many attempts were allowed for this test
+ self.max_allowed_attempts = max_allowed_attempts
def addMetric(self, name, value):
"""
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 16e9c7fbf45c5..73db67aedb739 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -2292,7 +2292,9 @@ def runOnce(
if err:
output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
- return lit.Test.Result(status, output)
+ return lit.Test.Result(
+ status, output, attempts=i + 1, max_allowed_attempts=attempts
+ )
def executeShTest(
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 3292554ab5ff7..6d98bb1c5ce56 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -104,6 +104,12 @@ def parse_args():
help="Do not use curses based progress bar",
action="store_false",
)
+ format_group.add_argument(
+ "--show-attempts-count",
+ dest="showAttemptsCount",
+ help="Show number of attempts and maximum attempts for flaky tests.",
+ action="store_true",
+ )
# Note: this does not generate flags for user-defined result codes.
success_codes = [c for c in lit.Test.ResultCode.all_codes() if not c.isFailure]
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 7de5a298d2302..399432e1ae8ac 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -117,9 +117,32 @@ def clear(self, interrupted):
def print_result(self, test):
# Show the test result line.
test_name = test.getFullName()
+
+ extra_info = ""
+ extra_attrs = []
+
+ if self.opts.showAttemptsCount:
+ if (
+ test.result.max_allowed_attempts is not None
+ and test.result.max_allowed_attempts > 1
+ ):
+ extra_attrs.append(f"attempts={test.result.attempts}")
+ extra_attrs.append(
+ f"max_allowed_attempts={test.result.max_allowed_attempts}"
+ )
+
+ if len(extra_attrs) > 0:
+ extra_info = " [" + ",".join(extra_attrs) + "]"
+
print(
- "%s: %s (%d of %d)"
- % (test.result.code.name, test_name, self.completed, self.num_tests)
+ "%s: %s (%d of %d)%s"
+ % (
+ test.result.code.name,
+ test_name,
+ self.completed,
+ self.num_tests,
+ extra_info,
+ )
)
# Show the test failure output, if requested.
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
index f2333a7de455a..4227b89a1b452 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 8
# RUN: "%python" "%s" "%counter"
import sys
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
index f2333a7de455a..cf3c5f849d1a3 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 10
# RUN: "%python" "%s" "%counter"
import sys
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg b/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
index 2279d293849a8..cf15d1f442afa 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
@@ -9,4 +9,4 @@ config.test_exec_root = None
config.substitutions.append(("%python", lit_config.params.get("python", "")))
config.substitutions.append(("%counter", lit_config.params.get("counter", "")))
-config.test_retry_attempts = 3
\ No newline at end of file
+config.test_retry_attempts = 9
\ No newline at end of file
diff --git a/llvm/utils/lit/tests/allow-retries.py b/llvm/utils/lit/tests/allow-retries.py
index ba95d34a09a4c..2750ddca1baab 100644
--- a/llvm/utils/lit/tests/allow-retries.py
+++ b/llvm/utils/lit/tests/allow-retries.py
@@ -72,49 +72,57 @@
# CHECK-TEST7: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
-# * The "--max-retries-per-test" is specified high enough (3).
+# * The "--max-retries-per-test" is specified high enough (7).
# * No ALLOW_RETRIES keyword is used in the test script.
# * No config.test_retry_attempts is adjusted in the test suite config file.
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \
-# RUN: --max-retries-per-test=3 \
+# RUN: --max-retries-per-test=7 \
+# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST8 %s
+# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=8]
# CHECK-TEST8: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (2).
-# * ALLOW_RETRIES is specified high enough (3)
+# * ALLOW_RETRIES is specified high enough (8)
# * No config.test_retry_attempts is adjusted in the test suite config file.
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=2 \
+# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST9 %s
+# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=9]
# CHECK-TEST9: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (2).
# * No ALLOW_RETRIES keyword is used in the test script.
-# * config.test_retry_attempts is set high enough (3).
+# * config.test_retry_attempts is set high enough (9).
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=2 \
+# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST10 %s
+# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=10]
# CHECK-TEST10: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (1).
-# * ALLOW_RETRIES keyword set high enough (3).
-# * config.test_retry_attempts is set too low enough (2).
+# * ALLOW_RETRIES keyword is set high enough (10)
+# * config.test_retry_attempts is set too low (2).
# RUN: rm -f %t.counter
-# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
+# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=1 \
+# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST11 %s
+# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=11]
# CHECK-TEST11: Passed With Retry: 1
>From 3c5fea2dd67b1e8dfbf8c8a6f226baf0f7a15e2f Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Mon, 2 Jun 2025 16:15:02 +0000
Subject: [PATCH 2/3] Fixup help text for --show-attempts-count
---
llvm/utils/lit/lit/cl_arguments.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 6d98bb1c5ce56..6ed9025fc856c 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -107,7 +107,8 @@ def parse_args():
format_group.add_argument(
"--show-attempts-count",
dest="showAttemptsCount",
- help="Show number of attempts and maximum attempts for flaky tests.",
+ help="Show number of attempts and maximum attempts for tests"
+ "which are allowed to run more than once.",
action="store_true",
)
>From abf946fa093b91f779c56c3d0f8caa50914ac6f4 Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Mon, 2 Jun 2025 16:15:54 +0000
Subject: [PATCH 3/3] Add missing leading whitespace to doc string
---
llvm/utils/lit/lit/cl_arguments.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 6ed9025fc856c..bdc3b1981f343 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -108,7 +108,7 @@ def parse_args():
"--show-attempts-count",
dest="showAttemptsCount",
help="Show number of attempts and maximum attempts for tests"
- "which are allowed to run more than once.",
+ " which are allowed to run more than once.",
action="store_true",
)
More information about the llvm-commits
mailing list