[llvm] 6918314 - [lit] show retry attempts (#142413)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 6 01:24:30 PDT 2025
Author: Konrad Kleine
Date: 2025-06-06T10:24:26+02:00
New Revision: 69183149189095966b9e7740d3b66126b754fbd1
URL: https://github.com/llvm/llvm-project/commit/69183149189095966b9e7740d3b66126b754fbd1
DIFF: https://github.com/llvm/llvm-project/commit/69183149189095966b9e7740d3b66126b754fbd1.diff
LOG: [lit] show retry attempts (#142413)
If a test took more than one attempt to complete, show the number of attempts and the maximum allowed attempts as `2 of 4 attempts` inside the `<progress info>` (see [TEST RUN OUTPUT FORMAT](https://llvm.org/docs/CommandGuide/lit.html#test-run-output-format)).
NOTE: Additionally this is a fixup for #141851 where the tests were not quite right. `max-retries-per-test/allow-retries-test_retry_attempts/test.py` was added but never used there. Now we're calling it. To correlate better between the test output and the test script I've used higher numbers of max allowed retries.
Added:
Modified:
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/Test.py
llvm/utils/lit/lit/TestRunner.py
llvm/utils/lit/lit/display.py
llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
llvm/utils/lit/tests/allow-retries.py
Removed:
################################################################################
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 1a9777cb3f544..938b7f9f8b6f3 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -688,6 +688,14 @@ newline.
The ``<progress info>`` field can be used to report progress information such
as (1/300) or can be empty, but even when empty the parentheses are required.
+Should a test be allowed retries (see ``ALLOW_RETRIES:`` annotation) and it
+needed more than one attempt to succeed, then ``<progress info>`` is extended
+by this information:
+
+.. code-block:: none
+
+ , <num_attempts_made> of <max_allowed_attempts> attempts
+
Each test result may include additional (multiline) log information in the
following format:
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 051062706f856..1bd5ba8dc2070 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -151,7 +151,9 @@ def toMetricValue(value):
class Result(object):
"""Wrapper for the results of executing an individual test."""
- def __init__(self, code, output="", elapsed=None):
+ def __init__(
+ self, code, output="", elapsed=None, attempts=1, max_allowed_attempts=None
+ ):
# The result code.
self.code = code
# The test output.
@@ -164,6 +166,10 @@ def __init__(self, code, output="", elapsed=None):
self.metrics = {}
# The micro-test results reported by this test.
self.microResults = {}
+ # How often was the test run?
+ self.attempts = attempts
+ # How many attempts were allowed for this test
+ self.max_allowed_attempts = max_allowed_attempts
def addMetric(self, name, value):
"""
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 16e9c7fbf45c5..73db67aedb739 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -2292,7 +2292,9 @@ def runOnce(
if err:
output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
- return lit.Test.Result(status, output)
+ return lit.Test.Result(
+ status, output, attempts=i + 1, max_allowed_attempts=attempts
+ )
def executeShTest(
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 7de5a298d2302..3a224f734bf71 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -117,9 +117,20 @@ def clear(self, interrupted):
def print_result(self, test):
# Show the test result line.
test_name = test.getFullName()
+
+ extra_info = ""
+ if test.result.attempts > 1:
+ extra_info = f", {test.result.attempts} of {test.result.max_allowed_attempts} attempts"
+
print(
- "%s: %s (%d of %d)"
- % (test.result.code.name, test_name, self.completed, self.num_tests)
+ "%s: %s (%d of %d%s)"
+ % (
+ test.result.code.name,
+ test_name,
+ self.completed,
+ self.num_tests,
+ extra_info,
+ )
)
# Show the test failure output, if requested.
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
index f2333a7de455a..4227b89a1b452 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 8
# RUN: "%python" "%s" "%counter"
import sys
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
index f2333a7de455a..cf3c5f849d1a3 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 10
# RUN: "%python" "%s" "%counter"
import sys
diff --git a/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg b/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
index 2279d293849a8..cf15d1f442afa 100644
--- a/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
+++ b/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
@@ -9,4 +9,4 @@ config.test_exec_root = None
config.substitutions.append(("%python", lit_config.params.get("python", "")))
config.substitutions.append(("%counter", lit_config.params.get("counter", "")))
-config.test_retry_attempts = 3
\ No newline at end of file
+config.test_retry_attempts = 9
\ No newline at end of file
diff --git a/llvm/utils/lit/tests/allow-retries.py b/llvm/utils/lit/tests/allow-retries.py
index ba95d34a09a4c..98eed142964a6 100644
--- a/llvm/utils/lit/tests/allow-retries.py
+++ b/llvm/utils/lit/tests/allow-retries.py
@@ -22,7 +22,7 @@
# RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py -v |\
# RUN: FileCheck --check-prefix=CHECK-TEST3 -match-full-lines %s
#
-# CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1)
+# CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1, 4 of 4 attempts)
# CHECK-TEST3-NEXT: {{\**}} TEST 'allow-retries :: does-not-succeed-within-limit.py' FAILED {{\**}}
# CHECK-TEST3-NEXT: Exit Code: 1
# CHECK-TEST3-EMPTY:
@@ -72,20 +72,21 @@
# CHECK-TEST7: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
-# * The "--max-retries-per-test" is specified high enough (3).
+# * The "--max-retries-per-test" is specified high enough (7).
# * No ALLOW_RETRIES keyword is used in the test script.
# * No config.test_retry_attempts is adjusted in the test suite config file.
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \
-# RUN: --max-retries-per-test=3 \
+# RUN: --max-retries-per-test=7 \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST8 %s
+# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 8 attempts)
# CHECK-TEST8: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (2).
-# * ALLOW_RETRIES is specified high enough (3)
+# * ALLOW_RETRIES is specified high enough (8)
# * No config.test_retry_attempts is adjusted in the test suite config file.
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \
@@ -93,28 +94,31 @@
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST9 %s
+# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 9 attempts)
# CHECK-TEST9: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (2).
# * No ALLOW_RETRIES keyword is used in the test script.
-# * config.test_retry_attempts is set high enough (3).
+# * config.test_retry_attempts is set high enough (9).
# RUN: rm -f %t.counter
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=2 \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST10 %s
+# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 10 attempts)
# CHECK-TEST10: Passed With Retry: 1
# This test only passes on the 4th try. Here we check that a test can be re-run when:
# * The "--max-retries-per-test" is specified too low (1).
-# * ALLOW_RETRIES keyword set high enough (3).
-# * config.test_retry_attempts is set too low enough (2).
+# * ALLOW_RETRIES keyword is set high enough (10)
+# * config.test_retry_attempts is set too low (2).
# RUN: rm -f %t.counter
-# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
+# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \
# RUN: --max-retries-per-test=1 \
# RUN: -Dcounter=%t.counter \
# RUN: -Dpython=%{python} \
# RUN: | FileCheck --check-prefix=CHECK-TEST11 %s
+# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 11 attempts)
# CHECK-TEST11: Passed With Retry: 1
More information about the llvm-commits
mailing list