[llvm] [lit] show retry attempts (PR #142413)
Konrad Kleine via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 5 23:09:19 PDT 2025
https://github.com/kwk updated https://github.com/llvm/llvm-project/pull/142413
>From ccffc2eccf52664b6eca8b8d6ab68f8b8a65de21 Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Mon, 2 Jun 2025 11:48:35 +0000
Subject: [PATCH 1/8] [lit] introduce --show-attempts-count
This introduces `--show-attempts-count` output option to `lit`.
It shows the number of attempts needed for a test to pass. It also shows
the maximum number of attempts that were allowed for this test. This option
is useful when you want to understand how many attempts took for a flaky
test (``FLAKYPASS``) to pass.
This is how the test output looks like *without* `--show-attempts-count`:
```
PASS: your-test-suite :: your-first-test.py (1 of 3)
FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2)
```
This is the output *with* `--show-attempts-count`:
```
PASS: your-test-suite :: your-first-test.py (1 of 2)
FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2) [attempts=3,max_allowed_attempts=4]
```
In this case `your-second-test.py` was executed three times and it
succeeded after the third time (`attempts=3`). Technically another run would have
been possible (`max_allowed_attempts=4`).
We will only append the extra information when a test was allowed more
than one attempt to succeed (i.e. see :option:`--max-retries-per-test`).
NOTE: Additionally this is a fixup for #141851 where the tests were not
quite right. `max-retries-per-test/allow-retries-test_retry_attempts/test.py`
was added but never used there. Now we're calling it. To correlate better between
the test output and the test script I've used higher numbers of max
allowed retries.
---
llvm/docs/CommandGuide/lit.rst | 28 +++++++++++++++++++
llvm/utils/lit/lit/Test.py | 8 +++++-
llvm/utils/lit/lit/TestRunner.py | 4 ++-
llvm/utils/lit/lit/cl_arguments.py | 6 ++++
llvm/utils/lit/lit/display.py | 27 ++++++++++++++++--
.../test.py | 2 +-
.../allow-retries-test_retry_attempts/test.py | 2 +-
.../lit.cfg | 2 +-
llvm/utils/lit/tests/allow-retries.py | 22 ++++++++++-----
9 files changed, 87 insertions(+), 14 deletions(-)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 86879f870e06e..73219cb7bfe98 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -144,6 +144,34 @@ OUTPUT OPTIONS
Show expectedly failed tests.
+.. option:: --show-attempts-count
+
+ Show the number of attempts needed for a test to pass. It also shows the
+ maximum number of attempts that were allowed for this test. This option is
+ useful when you want to understand how many attempts took for a flaky test
+ (``FLAKYPASS``) to pass.
+
+ This is how the test output looks like *without* :option:`--show-attempts-count`
+
+ .. code-block:: none
+
+ PASS: your-test-suite :: your-first-test.py (1 of 3)
+ FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2)
+
+ This is the output *with* :option:`--show-attempts-count`
+
+ .. code-block:: none
+
+ PASS: your-test-suite :: your-first-test.py (1 of 2)
+ FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2) [attempts=3,max_allowed_attempts=4]
+
+ In this case ``your-second-test.py`` was executed three times and it succeeded
+ after the third time (``attempts=3``). Technically another run would have been
+ possible (``max_allowed_attempts=4``).
+
+ We will only append the extra information when a test was allowed more than one
+ attempt to succeed (i.e. see :option:`--max-retries-per-test`).
+
.. _execution-options:
EXECUTION OPTIONS
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 051062706f856..fd8fc4c5f89ce 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -151,7 +151,9 @@ def toMetricValue(value):
class Result(object):
"""Wrapper for the results of executing an individual test."""
- def __init__(self, code, output="", elapsed=None):
+ def __init__(
+ self, code, output="", elapsed=None, attempts=None, max_allowed_attempts=None
+ ):
# The result code.
self.code = code
# The test output.
@@ -164,6 +166,10 @@ def __init__(self, code, output="", elapsed=None):
self.metrics = {}
# The micro-test results reported by this test.
self.microResults = {}
+ # How often was the test run?
+ self.attempts = attempts
+ # How many attempts were allowed for this test
+ self.max_allowed_attempts = max_allowed_attempts
def addMetric(self, name, value):
"""
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 16e9c7fbf45c5..73db67aedb739 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -2292,7 +2292,9 @@ def runOnce(
if err:
output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
- return lit.Test.Result(status, output)
+ return lit.Test.Result(
+ status, output, attempts=i + 1, max_allowed_attempts=attempts
+ )
def executeShTest(
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 3292554ab5ff7..6d98bb1c5ce56 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -104,6 +104,12 @@ def parse_args():
help="Do not use curses based progress bar",
action="store_false",
)
+ format_group.add_argument(
+ "--show-attempts-count",
+ dest="showAttemptsCount",
+ help="Show number of attempts and maximum attempts for flaky tests.",
+ action="store_true",
+ )
# Note: this does not generate flags for user-defined result codes.
success_codes = [c for c in lit.Test.ResultCode.all_codes() if not c.isFailure]
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 7de5a298d2302..399432e1ae8ac 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -117,9 +117,32 @@ def clear(self, interrupted):
def print_result(self, test):
# Show the test result line.
test_name = test.getFullName()
+
+ extra_info = ""
+ extra_attrs = []
+
+ if self.opts.showAttemptsCount:
+ if (
+ test.result.max_allowed_attempts is not None
+ and test.result.max_allowed_attempts > 1
+ ):
+ extra_attrs.append(f"attempts={test.result.attempts}")
+ extra_attrs.append(
+ f"max_allowed_attempts={test.result.max_allowed_attempts}"
+ )
+
+ if len(extra_attrs) > 0:
+ extra_info = " [" + ",".join(extra_attrs) + "]"
+
print(
- "%s: %s (%d of %d)"
- % (test.result.code.name, test_name, self.completed, self.num_tests)
+ "%s: %s (%d of %d)%s"
+ % (
+ test.result.code.name,
+ test_name,
+ self.completed,
+ self.num_tests,
+ extra_info,
+ )
)
# Show the test failure output, if requested.
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
index f2333a7de455a..4227b89a1b452 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 8
# RUN: "%python" "%s" "%counter"
import sys
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
index f2333a7de455a..cf3c5f849d1a3 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 10
# RUN: "%python" "%s" "%counter"
import sys
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg b/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
index 2279d293849a8..cf15d1f442afa 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
@@ -9,4 +9,4 @@ config.test_exec_root = None
config.substitutions.append(("%python", lit_config.params.get("python", "")))
config.substitutions.append(("%counter", lit_config.params.get("counter", "")))
-config.test_retry_attempts = 3
\ No newline at end of file
+config.test_retry_attempts = 9
\ No newline at end of file
diff --git a/llvm/utils/lit/tests/allow-retries.py b/llvm/utils/lit/tests/allow-retries.py
index ba95d34a09a4c..2750ddca1baab 100644
--- a/llvm/utils/lit/tests/allow-retries.py
+++ b/llvm/utils/lit/tests/allow-retries.py
@@ -72,49 +72,57 @@
# CHECK-TEST7: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
-# * The "--max-retries-per-test" is specified high enough (3).
+# * The "--max-retries-per-test" is specified high enough (7).
# * No ALLOW_RETRIES keyword is used in the test script.
# * No config.test_retry_attempts is adjusted in the test suite config file.
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \
-# RUN: --max-retries-per-test=3 \
+# RUN: --max-retries-per-test=7 \
+# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST8 %s
+# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=8]
# CHECK-TEST8: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (2).
-# * ALLOW_RETRIES is specified high enough (3)
+# * ALLOW_RETRIES is specified high enough (8)
# * No config.test_retry_attempts is adjusted in the test suite config file.
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=2 \
+# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST9 %s
+# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=9]
# CHECK-TEST9: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (2).
# * No ALLOW_RETRIES keyword is used in the test script.
-# * config.test_retry_attempts is set high enough (3).
+# * config.test_retry_attempts is set high enough (9).
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=2 \
+# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST10 %s
+# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=10]
# CHECK-TEST10: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (1).
-# * ALLOW_RETRIES keyword set high enough (3).
-# * config.test_retry_attempts is set too low enough (2).
+# * ALLOW_RETRIES keyword is set high enough (10)
+# * config.test_retry_attempts is set too low (2).
# RUN: rm -f %t.counter
-# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
+# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=1 \
+# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST11 %s
+# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=11]
# CHECK-TEST11: Passed With Retry: 1
>From 3c5fea2dd67b1e8dfbf8c8a6f226baf0f7a15e2f Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Mon, 2 Jun 2025 16:15:02 +0000
Subject: [PATCH 2/8] Fixup help text for --show-attempts-count
---
llvm/utils/lit/lit/cl_arguments.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 6d98bb1c5ce56..6ed9025fc856c 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -107,7 +107,8 @@ def parse_args():
format_group.add_argument(
"--show-attempts-count",
dest="showAttemptsCount",
- help="Show number of attempts and maximum attempts for flaky tests.",
+ help="Show number of attempts and maximum attempts for tests"
+ "which are allowed to run more than once.",
action="store_true",
)
>From abf946fa093b91f779c56c3d0f8caa50914ac6f4 Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Mon, 2 Jun 2025 16:15:54 +0000
Subject: [PATCH 3/8] Add missing leading whitespace to doc string
---
llvm/utils/lit/lit/cl_arguments.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 6ed9025fc856c..bdc3b1981f343 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -108,7 +108,7 @@ def parse_args():
"--show-attempts-count",
dest="showAttemptsCount",
help="Show number of attempts and maximum attempts for tests"
- "which are allowed to run more than once.",
+ " which are allowed to run more than once.",
action="store_true",
)
>From ea8aee56f381aecfb8dcbecc50d721598f7dc9ad Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Wed, 4 Jun 2025 15:33:31 +0000
Subject: [PATCH 4/8] Remove --show-attempts-count and make it default
---
llvm/docs/CommandGuide/lit.rst | 36 ++++++---------------------
llvm/utils/lit/lit/cl_arguments.py | 7 ------
llvm/utils/lit/lit/display.py | 21 +++++-----------
llvm/utils/lit/tests/allow-retries.py | 14 ++++-------
4 files changed, 19 insertions(+), 59 deletions(-)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 73219cb7bfe98..d9b5c655b921d 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -144,34 +144,6 @@ OUTPUT OPTIONS
Show expectedly failed tests.
-.. option:: --show-attempts-count
-
- Show the number of attempts needed for a test to pass. It also shows the
- maximum number of attempts that were allowed for this test. This option is
- useful when you want to understand how many attempts took for a flaky test
- (``FLAKYPASS``) to pass.
-
- This is how the test output looks like *without* :option:`--show-attempts-count`
-
- .. code-block:: none
-
- PASS: your-test-suite :: your-first-test.py (1 of 3)
- FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2)
-
- This is the output *with* :option:`--show-attempts-count`
-
- .. code-block:: none
-
- PASS: your-test-suite :: your-first-test.py (1 of 2)
- FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2) [attempts=3,max_allowed_attempts=4]
-
- In this case ``your-second-test.py`` was executed three times and it succeeded
- after the third time (``attempts=3``). Technically another run would have been
- possible (``max_allowed_attempts=4``).
-
- We will only append the extra information when a test was allowed more than one
- attempt to succeed (i.e. see :option:`--max-retries-per-test`).
-
.. _execution-options:
EXECUTION OPTIONS
@@ -716,6 +688,14 @@ newline.
The ``<progress info>`` field can be used to report progress information such
as (1/300) or can be empty, but even when empty the parentheses are required.
+Should a test be allowed retries (see ``ALLOW_RETRIES:`` annotation) and it
+needed more than one attempt to succeed, then ``<progess info>`` is extended
+by this information:
+
+.. code-block:: none
+
+ , <num_attempts_made> of <max_allowed_attempts> attempts
+
Each test result may include additional (multiline) log information in the
following format:
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index bdc3b1981f343..3292554ab5ff7 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -104,13 +104,6 @@ def parse_args():
help="Do not use curses based progress bar",
action="store_false",
)
- format_group.add_argument(
- "--show-attempts-count",
- dest="showAttemptsCount",
- help="Show number of attempts and maximum attempts for tests"
- " which are allowed to run more than once.",
- action="store_true",
- )
# Note: this does not generate flags for user-defined result codes.
success_codes = [c for c in lit.Test.ResultCode.all_codes() if not c.isFailure]
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 399432e1ae8ac..af4dab3b62d5a 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -119,23 +119,14 @@ def print_result(self, test):
test_name = test.getFullName()
extra_info = ""
- extra_attrs = []
-
- if self.opts.showAttemptsCount:
- if (
- test.result.max_allowed_attempts is not None
- and test.result.max_allowed_attempts > 1
- ):
- extra_attrs.append(f"attempts={test.result.attempts}")
- extra_attrs.append(
- f"max_allowed_attempts={test.result.max_allowed_attempts}"
- )
-
- if len(extra_attrs) > 0:
- extra_info = " [" + ",".join(extra_attrs) + "]"
+ if (
+ test.result.max_allowed_attempts is not None
+ and test.result.max_allowed_attempts > 1
+ ) and test.result.attempts > 0:
+ extra_info = f", {test.result.attempts} of {test.result.max_allowed_attempts} attempts"
print(
- "%s: %s (%d of %d)%s"
+ "%s: %s (%d of %d%s)"
% (
test.result.code.name,
test_name,
diff --git a/llvm/utils/lit/tests/allow-retries.py b/llvm/utils/lit/tests/allow-retries.py
index 2750ddca1baab..98eed142964a6 100644
--- a/llvm/utils/lit/tests/allow-retries.py
+++ b/llvm/utils/lit/tests/allow-retries.py
@@ -22,7 +22,7 @@
# RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py -v |\
# RUN: FileCheck --check-prefix=CHECK-TEST3 -match-full-lines %s
#
-# CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1)
+# CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1, 4 of 4 attempts)
# CHECK-TEST3-NEXT: {{\**}} TEST 'allow-retries :: does-not-succeed-within-limit.py' FAILED {{\**}}
# CHECK-TEST3-NEXT: Exit Code: 1
# CHECK-TEST3-EMPTY:
@@ -78,11 +78,10 @@
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=7 \
-# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST8 %s
-# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=8]
+# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 8 attempts)
# CHECK-TEST8: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
@@ -92,11 +91,10 @@
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=2 \
-# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST9 %s
-# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=9]
+# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 9 attempts)
# CHECK-TEST9: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
@@ -106,11 +104,10 @@
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=2 \
-# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST10 %s
-# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=10]
+# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 10 attempts)
# CHECK-TEST10: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
@@ -120,9 +117,8 @@
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=1 \
-# RUN: --show-attempts-count \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST11 %s
-# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=11]
+# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 11 attempts)
# CHECK-TEST11: Passed With Retry: 1
>From b6179fbff6544b25e5a7615ccae077dcd4635637 Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Wed, 4 Jun 2025 15:47:43 +0000
Subject: [PATCH 5/8] Fixup
---
llvm/utils/lit/lit/display.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index af4dab3b62d5a..8cfc19f948e37 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -122,7 +122,7 @@ def print_result(self, test):
if (
test.result.max_allowed_attempts is not None
and test.result.max_allowed_attempts > 1
- ) and test.result.attempts > 0:
+ ) and test.result.attempts > 1:
extra_info = f", {test.result.attempts} of {test.result.max_allowed_attempts} attempts"
print(
>From 5862756ff6e065cc35004a756ccd96358875bfd9 Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Thu, 5 Jun 2025 14:07:33 +0000
Subject: [PATCH 6/8] Fix typo: progess -> progress
---
llvm/docs/CommandGuide/lit.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index d9b5c655b921d..47e225d7306a7 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -689,7 +689,7 @@ The ``<progress info>`` field can be used to report progress information such
as (1/300) or can be empty, but even when empty the parentheses are required.
Should a test be allowed retries (see ``ALLOW_RETRIES:`` annotation) and it
-needed more than one attempt to succeed, then ``<progess info>`` is extended
+needed more than one attempt to succeed, then ``<progress info>`` is extended
by this information:
.. code-block:: none
>From 1eb0b45e23c8457f6864b0d5cd0994efd865f7a2 Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Thu, 5 Jun 2025 15:21:38 +0000
Subject: [PATCH 7/8] Less overprotection
---
llvm/utils/lit/lit/display.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 8cfc19f948e37..3a224f734bf71 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -119,10 +119,7 @@ def print_result(self, test):
test_name = test.getFullName()
extra_info = ""
- if (
- test.result.max_allowed_attempts is not None
- and test.result.max_allowed_attempts > 1
- ) and test.result.attempts > 1:
+ if test.result.attempts > 1:
extra_info = f", {test.result.attempts} of {test.result.max_allowed_attempts} attempts"
print(
>From 66ec87e265aabeb5d763518ef22538e572898a9c Mon Sep 17 00:00:00 2001
From: Konrad Kleine <kkleine at redhat.com>
Date: Fri, 6 Jun 2025 06:07:57 +0000
Subject: [PATCH 8/8] Assume a test always is allowed 1 attempt instead of None
---
llvm/utils/lit/lit/Test.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index fd8fc4c5f89ce..1bd5ba8dc2070 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -152,7 +152,7 @@ class Result(object):
"""Wrapper for the results of executing an individual test."""
def __init__(
- self, code, output="", elapsed=None, attempts=None, max_allowed_attempts=None
+ self, code, output="", elapsed=None, attempts=1, max_allowed_attempts=None
):
# The result code.
self.code = code
More information about the llvm-commits
mailing list