[llvm] [CI] Generate Test Report With No Test Results (PR #147871)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 10 06:41:33 PDT 2025
https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/147871
>From 03562806d878ba564143d8030b9bc36815f0cd5a Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Thu, 10 Jul 2025 01:52:25 +0000
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
=?UTF-8?q?anges=20to=20main=20this=20commit=20is=20based=20on?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.4
[skip ci]
---
.ci/generate_test_report_github.py | 4 +--
.ci/generate_test_report_lib.py | 30 ++++--------------
.ci/generate_test_report_lib_test.py | 47 +++++++---------------------
3 files changed, 20 insertions(+), 61 deletions(-)
diff --git a/.ci/generate_test_report_github.py b/.ci/generate_test_report_github.py
index 5f88513e92a69..be43c4b4f693f 100644
--- a/.ci/generate_test_report_github.py
+++ b/.ci/generate_test_report_github.py
@@ -16,8 +16,8 @@
parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
args = parser.parse_args()
- report, _ = generate_test_report_lib.generate_report_from_files(
- args.title, args.return_code, args.junit_files, None
+ report = generate_test_report_lib.generate_report_from_files(
+ args.title, args.return_code, args.junit_files
)
print(report)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 1d34d57e80506..dd203b5169d9f 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -18,7 +18,6 @@ def generate_report(
junit_objects,
size_limit=1024 * 1024,
list_failures=True,
- buildkite_info=None,
):
if not junit_objects:
# Note that we do not post an empty report, therefore we can ignore a
@@ -26,7 +25,7 @@ def generate_report(
#
# If we were going to post a report, then yes, it would be misleading
# to say we succeeded when the final return code was non-zero.
- return ("", "success")
+ return ""
failures = {}
tests_run = 0
@@ -52,12 +51,7 @@ def generate_report(
)
if not tests_run:
- return ("", None)
-
- style = "success"
- # Either tests failed, or all tests passed but something failed to build.
- if tests_failed or return_code != 0:
- style = "error"
+ return ""
report = [f"# {title}", ""]
@@ -73,22 +67,12 @@ def plural(num_tests):
if tests_failed:
report.append(f"* {tests_failed} {plural(tests_failed)} failed")
- if buildkite_info is not None:
- log_url = (
- "https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/"
- "pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/"
- "jobs/{BUILDKITE_JOB_ID}/download.txt".format(**buildkite_info)
- )
- download_text = f"[Download]({log_url})"
- else:
- download_text = "Download"
-
if not list_failures:
report.extend(
[
"",
"Failed tests and their output was too large to report. "
- f"{download_text} the build's log file to see the details.",
+ "Download the build's log file to see the details.",
]
)
elif failures:
@@ -118,7 +102,7 @@ def plural(num_tests):
"",
"All tests passed but another part of the build **failed**.",
"",
- f"{download_text} the build's log file to see the details.",
+ "Download the build's log file to see the details.",
]
)
@@ -141,16 +125,14 @@ def plural(num_tests):
junit_objects,
size_limit,
list_failures=False,
- buildkite_info=buildkite_info,
)
- return report, style
+ return report
-def generate_report_from_files(title, return_code, junit_files, buildkite_info):
+def generate_report_from_files(title, return_code, junit_files):
return generate_report(
title,
return_code,
[JUnitXml.fromfile(p) for p in junit_files],
- buildkite_info=buildkite_info,
)
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 38ac24afd231c..89dfed97a3596 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -20,9 +20,7 @@ def junit_from_xml(xml):
class TestReports(unittest.TestCase):
def test_title_only(self):
- self.assertEqual(
- generate_test_report_lib.generate_report("Foo", 0, []), ("", "success")
- )
+ self.assertEqual(generate_test_report_lib.generate_report("Foo", 0, []), "")
def test_no_tests_in_testsuite(self):
self.assertEqual(
@@ -42,7 +40,7 @@ def test_no_tests_in_testsuite(self):
)
],
),
- ("", None),
+ "",
)
def test_no_failures(self):
@@ -70,8 +68,7 @@ def test_no_failures(self):
# Foo
* 1 test passed"""
- ),
- "success",
+ )
),
)
@@ -93,12 +90,6 @@ def test_no_failures_build_failed(self):
)
)
],
- buildkite_info={
- "BUILDKITE_ORGANIZATION_SLUG": "organization_slug",
- "BUILDKITE_PIPELINE_SLUG": "pipeline_slug",
- "BUILDKITE_BUILD_NUMBER": "build_number",
- "BUILDKITE_JOB_ID": "job_id",
- },
),
(
dedent(
@@ -109,11 +100,10 @@ def test_no_failures_build_failed(self):
All tests passed but another part of the build **failed**.
- [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details.
+ Download the build's log file to see the details.
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
- ),
- "error",
+ )
),
)
@@ -174,14 +164,12 @@ def test_report_single_file_single_testsuite(self):
</details>
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
- ),
- "error",
+ )
),
)
- MULTI_SUITE_OUTPUT = (
- dedent(
- """\
+ MULTI_SUITE_OUTPUT = dedent(
+ """\
# ABC and DEF
* 1 test passed
@@ -210,8 +198,6 @@ def test_report_single_file_single_testsuite(self):
</details>
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
- ),
- "error",
)
def test_report_single_file_multiple_testsuites(self):
@@ -320,8 +306,7 @@ def test_report_dont_list_failures(self):
Failed tests and their output was too large to report. Download the build's log file to see the details.
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
- ),
- "error",
+ )
),
)
@@ -346,12 +331,6 @@ def test_report_dont_list_failures_link_to_log(self):
)
],
list_failures=False,
- buildkite_info={
- "BUILDKITE_ORGANIZATION_SLUG": "organization_slug",
- "BUILDKITE_PIPELINE_SLUG": "pipeline_slug",
- "BUILDKITE_BUILD_NUMBER": "build_number",
- "BUILDKITE_JOB_ID": "job_id",
- },
),
(
dedent(
@@ -360,11 +339,10 @@ def test_report_dont_list_failures_link_to_log(self):
* 1 test failed
- Failed tests and their output was too large to report. [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details.
+ Failed tests and their output was too large to report. Download the build's log file to see the details.
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
- ),
- "error",
+ )
),
)
@@ -403,7 +381,6 @@ def test_report_size_limit(self):
Failed tests and their output was too large to report. Download the build's log file to see the details.
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
- ),
- "error",
+ )
),
)
>From 8e7005442901f2c384042c45e6f912db4832329e Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Thu, 10 Jul 2025 13:41:21 +0000
Subject: [PATCH 2/2] feedback
Created using spr 1.3.4
---
.ci/generate_test_report_lib.py | 7 ++++++-
.ci/generate_test_report_lib_test.py | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index eed1e73fb44a8..25d810f1c6d17 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -54,7 +54,12 @@ def generate_report(
if tests_run == 0:
if return_code == 0:
- report.extend(["The build succeeded but no tests ran."])
+ report.extend(
+ [
+ "The build succeeded and no tests ran. This is expected in some "
+ "build configurations."
+ ]
+ )
else:
report.extend(
[
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 5e75b4ee8b297..eda76ead19b9d 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -26,7 +26,7 @@ def test_title_only(self):
"""\
# Foo
- The build succeeded but no tests ran."""
+ The build succeeded and no tests ran. This is expected in some build configurations."""
),
)
More information about the llvm-commits
mailing list