[llvm] [CI] Enable Build Failure Reporting (PR #152622)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 8 09:44:55 PDT 2025
https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/152622
>From af808e6f84da22725c31b7b2cce50572c40cfc9d Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 01:47:11 +0000
Subject: [PATCH 1/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
=?UTF-8?q?anges=20to=20main=20this=20commit=20is=20based=20on?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
[skip ci]
---
.ci/generate_test_report_lib.py | 135 ++++++++++++++---
.ci/generate_test_report_lib_test.py | 217 ++++++++++++++++++++++++++-
2 files changed, 330 insertions(+), 22 deletions(-)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 25d810f1c6d17..12bc1f18d1401 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -12,6 +12,57 @@
"https://github.com/llvm/llvm-project/issues and add the "
"`infrastructure` label."
)
+# The maximum number of lines to pull from a ninja failure.
+NINJA_LOG_SIZE_THRESHOLD = 500
+
+
+def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
+ """Parses an individual ninja log."""
+ failures = []
+ index = 0
+ while index < len(ninja_log):
+ while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):
+ index += 1
+ if index == len(ninja_log):
+ # We hit the end of the log without finding a build failure, go to
+ # the next log.
+ return failures
+ failing_action = ninja_log[index - 1].split("] ")[1]
+ failure_log = []
+ while (
+ index < len(ninja_log)
+ and not ninja_log[index].startswith("[")
+ and not ninja_log[index].startswith(
+ "ninja: build stopped: subcommand failed"
+ )
+ and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
+ ):
+ failure_log.append(ninja_log[index])
+ index += 1
+ failures.append((failing_action, "\n".join(failure_log)))
+ return failures
+
+
+def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
+ """Extracts failure messages from ninja output.
+
+ This patch takes stdout/stderr from ninja in the form of a list of files
+ represented as a list of lines. This function then returns tuples containing
+ the name of the target and the error message.
+
+ Args:
+ ninja_logs: A list of files in the form of a list of lines representing the log
+ files captured from ninja.
+
+ Returns:
+ A list of tuples. The first string is the name of the target that failed. The
+ second string is the error message.
+ """
+ failures = []
+ for ninja_log in ninja_logs:
+ log_failures = _parse_ninja_log(ninja_log)
+ failures.extend(log_failures)
+ return failures
# Set size_limit to limit the byte size of the report. The default is 1MB as this
@@ -24,6 +75,7 @@ def generate_report(
title,
return_code,
junit_objects,
+ ninja_logs: list[list[str]],
size_limit=1024 * 1024,
list_failures=True,
):
@@ -61,15 +113,46 @@ def generate_report(
]
)
else:
- report.extend(
- [
- "The build failed before running any tests.",
- "",
- SEE_BUILD_FILE_STR,
- "",
- UNRELATED_FAILURES_STR,
- ]
- )
+ ninja_failures = find_failure_in_ninja_logs(ninja_logs)
+ if not ninja_failures:
+ report.extend(
+ [
+ "The build failed before running any tests. Detailed "
+ "information about the build failure could not be "
+ "automatically obtained.",
+ "",
+ SEE_BUILD_FILE_STR,
+ "",
+ UNRELATED_FAILURES_STR,
+ ]
+ )
+ else:
+ report.extend(
+ [
+ "The build failed before running any tests. Click on the "
+ "failure below to see the details.",
+ "",
+ ]
+ )
+ for build_failure in ninja_failures:
+ failed_action, failure_message = build_failure
+ report.extend(
+ [
+ "<details>",
+ f"<summary>{failed_action}</summary>",
+ "",
+ "```",
+ failure_message,
+ "```",
+ "</details>",
+ ]
+ )
+ report.extend(
+ [
+ "",
+ UNRELATED_FAILURES_STR,
+ ]
+ )
return "\n".join(report)
tests_passed = tests_run - tests_skipped - tests_failed
@@ -114,14 +197,32 @@ def plural(num_tests):
elif return_code != 0:
# No tests failed but the build was in a failed state. Bring this to the user's
# attention.
- report.extend(
- [
- "",
- "All tests passed but another part of the build **failed**.",
- "",
- SEE_BUILD_FILE_STR,
- ]
- )
+ ninja_failures = find_failure_in_ninja_logs(ninja_logs)
+ if not ninja_failures:
+ report.extend(
+ [
+ "",
+ "All tests passed but another part of the build **failed**. "
+ "Detailed information about the build failure could not be "
+ "automatically obtained.",
+ "",
+ SEE_BUILD_FILE_STR,
+ ]
+ )
+ else:
+ for build_failure in ninja_failures:
+ failed_action, failure_message = build_failure
+ report.extend(
+ [
+ "<details>",
+ f"<summary>{failed_action}</summary>",
+ "",
+ "```",
+ failure_message,
+ "```",
+ "</details>",
+ ]
+ )
if failures or return_code != 0:
report.extend(["", UNRELATED_FAILURES_STR])
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index eda76ead19b9d..057f9f7506c74 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -19,9 +19,116 @@ def junit_from_xml(xml):
class TestReports(unittest.TestCase):
+ def test_find_failure_ninja_logs(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: test/4.stamp",
+ "touch test/4.stamp",
+ "Wow! This system is really broken!",
+ "[5/5] test/5.stamp",
+ ],
+ ]
+ )
+ self.assertEqual(len(failures), 1)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/4.stamp",
+ dedent(
+ """\
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+
+ def test_no_failure_ninja_log(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/3] test/1.stamp",
+ "[2/3] test/2.stamp",
+ "[3/3] test/3.stamp",
+ ]
+ ]
+ )
+ self.assertEqual(failures, [])
+
+ def test_ninja_log_end(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/3] test/1.stamp",
+ "[2/3] test/2.stamp",
+ "[3/3] test/3.stamp",
+ "FAILED: touch test/3.stamp",
+ "Wow! This system is really broken!",
+ "ninja: build stopped: subcommand failed.",
+ ]
+ ]
+ )
+ self.assertEqual(len(failures), 1)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/3.stamp",
+ dedent(
+ """\
+ FAILED: touch test/3.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+
+ def test_ninja_log_multiple_failures(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! This system is really broken!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! This system is maybe broken!",
+ "[5/5] test/5.stamp",
+ ]
+ ]
+ )
+ self.assertEqual(len(failures), 2)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/2.stamp",
+ dedent(
+ """\
+ FAILED: touch test/2.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+ self.assertEqual(
+ failures[1],
+ (
+ "test/4.stamp",
+ dedent(
+ """\
+ FAILED: touch test/4.stamp
+ Wow! This system is maybe broken!"""
+ ),
+ ),
+ )
+
def test_title_only(self):
self.assertEqual(
- generate_test_report_lib.generate_report("Foo", 0, []),
+ generate_test_report_lib.generate_report("Foo", 0, [], []),
dedent(
"""\
# Foo
@@ -32,12 +139,12 @@ def test_title_only(self):
def test_title_only_failure(self):
self.assertEqual(
- generate_test_report_lib.generate_report("Foo", 1, []),
+ generate_test_report_lib.generate_report("Foo", 1, [], []),
dedent(
"""\
# Foo
- The build failed before running any tests.
+ The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
Download the build's log file to see the details.
@@ -45,6 +152,45 @@ def test_title_only_failure(self):
),
)
+ def test_title_only_failure_ninja_log(self):
+ self.assertEqual(
+ generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: test/4.stamp",
+ "touch test/4.stamp",
+ "Wow! Risk!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ ),
+ dedent(
+ """\
+ # Foo
+
+ The build failed before running any tests. Click on the failure below to see the details.
+
+ <details>
+ <summary>test/4.stamp</summary>
+
+ ```
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! Risk!
+ ```
+ </details>
+
+ If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+ ),
+ )
+
def test_no_tests_in_testsuite(self):
self.assertEqual(
generate_test_report_lib.generate_report(
@@ -62,12 +208,13 @@ def test_no_tests_in_testsuite(self):
)
)
],
+ [],
),
dedent(
"""\
# Foo
- The build failed before running any tests.
+ The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
Download the build's log file to see the details.
@@ -93,6 +240,7 @@ def test_no_failures(self):
)
)
],
+ [],
),
(
dedent(
@@ -122,6 +270,7 @@ def test_no_failures_build_failed(self):
)
)
],
+ [],
),
(
dedent(
@@ -130,7 +279,7 @@ def test_no_failures_build_failed(self):
* 1 test passed
- All tests passed but another part of the build **failed**.
+ All tests passed but another part of the build **failed**. Detailed information about the build failure could not be automatically obtained.
Download the build's log file to see the details.
@@ -139,6 +288,58 @@ def test_no_failures_build_failed(self):
),
)
+ def test_no_failures_build_failed_ninja_log(self):
+ self.assertEqual(
+ generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.00">
+ <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+ <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: test/4.stamp",
+ "touch test/4.stamp",
+ "Wow! Close To You!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ ),
+ (
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed
+ <details>
+ <summary>test/4.stamp</summary>
+
+ ```
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! Close To You!
+ ```
+ </details>
+
+ If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+ )
+ ),
+ )
+
def test_report_single_file_single_testsuite(self):
self.assertEqual(
generate_test_report_lib.generate_report(
@@ -166,6 +367,7 @@ def test_report_single_file_single_testsuite(self):
)
)
],
+ [],
),
(
dedent(
@@ -261,6 +463,7 @@ def test_report_single_file_multiple_testsuites(self):
)
)
],
+ [],
),
self.MULTI_SUITE_OUTPUT,
)
@@ -302,6 +505,7 @@ def test_report_multiple_files_multiple_testsuites(self):
)
),
],
+ [],
),
self.MULTI_SUITE_OUTPUT,
)
@@ -326,6 +530,7 @@ def test_report_dont_list_failures(self):
)
)
],
+ [],
list_failures=False,
),
(
@@ -362,6 +567,7 @@ def test_report_dont_list_failures_link_to_log(self):
)
)
],
+ [],
list_failures=False,
),
(
@@ -401,6 +607,7 @@ def test_report_size_limit(self):
)
)
],
+ [],
size_limit=512,
),
(
>From cff6dfd1c4ee6c0c7177b6dc11946b45b50fc009 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:02:07 +0000
Subject: [PATCH 2/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
=?UTF-8?q?anges=20introduced=20through=20rebase?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
[skip ci]
---
.ci/generate_test_report_lib.py | 63 +++++++++++++---------------
.ci/generate_test_report_lib_test.py | 10 ++---
2 files changed, 34 insertions(+), 39 deletions(-)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 12bc1f18d1401..f62ab817fdb50 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -27,14 +27,16 @@ def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
# We hit the end of the log without finding a build failure, go to
# the next log.
return failures
+ # index will point to the line that starts with Failed:. The progress
+ # indicator is the line before this and contains a pretty printed version
+ # of the target being built. We use this and remove the progress information
+ # to get a succinct name for the target.
failing_action = ninja_log[index - 1].split("] ")[1]
failure_log = []
while (
index < len(ninja_log)
and not ninja_log[index].startswith("[")
- and not ninja_log[index].startswith(
- "ninja: build stopped: subcommand failed"
- )
+ and not ninja_log[index].startswith("ninja: build stopped:")
and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
):
failure_log.append(ninja_log[index])
@@ -46,7 +48,7 @@ def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
"""Extracts failure messages from ninja output.
- This patch takes stdout/stderr from ninja in the form of a list of files
+ This function takes stdout/stderr from ninja in the form of a list of files
represented as a list of lines. This function then returns tuples containing
the name of the target and the error message.
@@ -65,6 +67,25 @@ def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, s
return failures
+def _format_ninja_failures(ninja_failures: list[tuple[str, str]]) -> list[str]:
+ """Formats ninja failures into summary views for the report."""
+ output = []
+ for build_failure in ninja_failures:
+ failed_action, failure_message = build_failure
+ output.extend(
+ [
+ "<details>",
+ f"<summary>{failed_action}</summary>",
+ "",
+ "```",
+ failure_message,
+ "```",
+ "</details>",
+ ]
+ )
+ return output
+
+
# Set size_limit to limit the byte size of the report. The default is 1MB as this
# is the most that can be put into an annotation. If the generated report exceeds
# this limit and failures are listed, it will be generated again without failures
@@ -129,24 +150,12 @@ def generate_report(
else:
report.extend(
[
- "The build failed before running any tests. Click on the "
+ "The build failed before running any tests. Click on a "
"failure below to see the details.",
"",
]
)
- for build_failure in ninja_failures:
- failed_action, failure_message = build_failure
- report.extend(
- [
- "<details>",
- f"<summary>{failed_action}</summary>",
- "",
- "```",
- failure_message,
- "```",
- "</details>",
- ]
- )
+ report.extend(_format_ninja_failures(ninja_failures))
report.extend(
[
"",
@@ -203,26 +212,14 @@ def plural(num_tests):
[
"",
"All tests passed but another part of the build **failed**. "
- "Detailed information about the build failure could not be "
- "automatically obtained.",
+ "Information about the build failure could not be automatically "
+ "obtained.",
"",
SEE_BUILD_FILE_STR,
]
)
else:
- for build_failure in ninja_failures:
- failed_action, failure_message = build_failure
- report.extend(
- [
- "<details>",
- f"<summary>{failed_action}</summary>",
- "",
- "```",
- failure_message,
- "```",
- "</details>",
- ]
- )
+ report.extend(_format_ninja_failures(ninja_failures))
if failures or return_code != 0:
report.extend(["", UNRELATED_FAILURES_STR])
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 057f9f7506c74..e07875a4bd10d 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -27,8 +27,7 @@ def test_find_failure_ninja_logs(self):
"[2/5] test/2.stamp",
"[3/5] test/3.stamp",
"[4/5] test/4.stamp",
- "FAILED: test/4.stamp",
- "touch test/4.stamp",
+ "FAILED: touch test/4.stamp",
"Wow! This system is really broken!",
"[5/5] test/5.stamp",
],
@@ -41,8 +40,7 @@ def test_find_failure_ninja_logs(self):
"test/4.stamp",
dedent(
"""\
- FAILED: test/4.stamp
- touch test/4.stamp
+ FAILED: touch test/4.stamp
Wow! This system is really broken!"""
),
),
@@ -175,7 +173,7 @@ def test_title_only_failure_ninja_log(self):
"""\
# Foo
- The build failed before running any tests. Click on the failure below to see the details.
+ The build failed before running any tests. Click on a failure below to see the details.
<details>
<summary>test/4.stamp</summary>
@@ -279,7 +277,7 @@ def test_no_failures_build_failed(self):
* 1 test passed
- All tests passed but another part of the build **failed**. Detailed information about the build failure could not be automatically obtained.
+ All tests passed but another part of the build **failed**. Information about the build failure could not be automatically obtained.
Download the build's log file to see the details.
>From 8ead35e5d0b9e42d112aba5b728ca73fb23b9769 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:03:12 +0000
Subject: [PATCH 3/9] feedback
Created using spr 1.3.6
---
.ci/generate_test_report_lib_test.py | 7 -------
1 file changed, 7 deletions(-)
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index bc96cc7dde554..6023b57ffa538 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -654,13 +654,6 @@ def test_generate_report_end_to_end(self):
[5/5] test/5.stamp"""
)
)
- test = generate_test_report_lib.generate_report_from_files(
- "Foo", 1, [junit_xml_file, ninja_log_file]
- )
-
- print(test)
- with open("/tmp/blah", "w") as handle2:
- handle2.write(test)
self.assertEqual(
generate_test_report_lib.generate_report_from_files(
"Foo", 1, [junit_xml_file, ninja_log_file]
>From 7deba5a9d218a5285a3b0c2d76af016158e694e8 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:28:24 +0000
Subject: [PATCH 4/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
=?UTF-8?q?anges=20introduced=20through=20rebase?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
[skip ci]
---
.ci/generate_test_report_lib.py | 8 ++++++++
.ci/generate_test_report_lib_test.py | 3 +++
2 files changed, 11 insertions(+)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index f62ab817fdb50..e4152ce54d352 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -219,6 +219,14 @@ def plural(num_tests):
]
)
else:
+ report.extend(
+ [
+ "",
+ "All tests passed but another part of the build **failed**. Click on "
+ "a failure below to see the details.",
+ "",
+ ]
+ )
report.extend(_format_ninja_failures(ninja_failures))
if failures or return_code != 0:
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index e07875a4bd10d..e91ea48c704c6 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -323,6 +323,9 @@ def test_no_failures_build_failed_ninja_log(self):
# Foo
* 1 test passed
+
+ All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
<details>
<summary>test/4.stamp</summary>
>From ae847e587b034fed4371578c28e3e07962eeaec7 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:35:05 +0000
Subject: [PATCH 5/9] feedback
Created using spr 1.3.6
---
.ci/generate_test_report_lib_test.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index e28449022c2f9..cd5af0381cce9 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -666,6 +666,9 @@ def test_generate_report_end_to_end(self):
# Foo
* 1 test passed
+
+ All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
<details>
<summary>test/4.stamp</summary>
>From 180401665b19db8f11acc7675337a6d64d0670c7 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:52:55 +0000
Subject: [PATCH 6/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
=?UTF-8?q?anges=20introduced=20through=20rebase?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
[skip ci]
---
.ci/generate_test_report_lib_test.py | 94 ++++++++++++++++++++++++++++
1 file changed, 94 insertions(+)
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index e91ea48c704c6..389d781042e23 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -341,6 +341,100 @@ def test_no_failures_build_failed_ninja_log(self):
),
)
+ def test_no_failures_multiple_build_failed_ninja_log(self):
+ test = generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.00">
+ <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+ <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! Be Kind!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! I Dare You!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ )
+ print(test)
+ self.assertEqual(
+ generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.00">
+ <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+ <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! Be Kind!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! I Dare You!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ ),
+ (
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed
+
+ All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+ <details>
+ <summary>test/2.stamp</summary>
+
+ ```
+ FAILED: touch test/2.stamp
+ Wow! Be Kind!
+ ```
+ </details>
+ <details>
+ <summary>test/4.stamp</summary>
+
+ ```
+ FAILED: touch test/4.stamp
+ Wow! I Dare You!
+ ```
+ </details>
+
+ If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+ )
+ ),
+ )
+
def test_report_single_file_single_testsuite(self):
self.assertEqual(
generate_test_report_lib.generate_report(
>From ce567ca77ffce0bb8b368c03d87dff6e401c5635 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 15:02:17 +0000
Subject: [PATCH 7/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
=?UTF-8?q?anges=20introduced=20through=20rebase?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
[skip ci]
---
.ci/generate_test_report_lib.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index e4152ce54d352..419913325a156 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -27,10 +27,16 @@ def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
# We hit the end of the log without finding a build failure, go to
# the next log.
return failures
+ # We are trying to parse cases like the following:
+ #
+ # [4/5] test/4.stamp
+ # FAILED: touch test/4.stamp
+ # touch test/4.stamp
+ #
# index will point to the line that starts with Failed:. The progress
- # indicator is the line before this and contains a pretty printed version
- # of the target being built. We use this and remove the progress information
- # to get a succinct name for the target.
+ # indicator is the line before this ([4/5] test/4.stamp) and contains a pretty
+ # printed version of the target being built (test/4.stamp). We use this line
+ # and remove the progress information to get a succinct name for the target.
failing_action = ninja_log[index - 1].split("] ")[1]
failure_log = []
while (
>From d9c5c3f34e8cb1e14740c93cc8656ea1f6582b9b Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 16:32:46 +0000
Subject: [PATCH 8/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
=?UTF-8?q?anges=20introduced=20through=20rebase?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
[skip ci]
---
.ci/generate_test_report_lib.py | 4 +---
.github/workflows/premerge.yaml | 9 +++++++++
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 419913325a156..75e3ca0e7d32d 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -253,7 +253,5 @@ def plural(num_tests):
def generate_report_from_files(title, return_code, junit_files):
return generate_report(
- title,
- return_code,
- [JUnitXml.fromfile(p) for p in junit_files],
+ title, return_code, [JUnitXml.fromfile(p) for p in junit_files], []
)
diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml
index d0518fa6879e2..9dbb2dfe66480 100644
--- a/.github/workflows/premerge.yaml
+++ b/.github/workflows/premerge.yaml
@@ -70,6 +70,12 @@ jobs:
export SCCACHE_IDLE_TIMEOUT=0
sccache --start-server
+ export projects_to_build=polly
+ export project_check_targets=check-polly
+ export runtimes_to_build=""
+ export runtimes_check_targets=""
+ export runtimes_check_targets_needs_reconfig=""
+
./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}" "${enable_cir}"
- name: Upload Artifacts
if: '!cancelled()'
@@ -106,6 +112,9 @@ jobs:
echo "Building projects: ${projects_to_build}"
echo "Running project checks targets: ${project_check_targets}"
+ export projects_to_build=polly
+ export project_check_targets=check-polly
+
echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT
echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT
- name: Build and Test
>From 95f505372320bdfd545af9727d697611c15148e3 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 16:42:50 +0000
Subject: [PATCH 9/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
=?UTF-8?q?anges=20introduced=20through=20rebase?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
[skip ci]
---
.github/workflows/premerge.yaml | 9 ---------
1 file changed, 9 deletions(-)
diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml
index 9dbb2dfe66480..d0518fa6879e2 100644
--- a/.github/workflows/premerge.yaml
+++ b/.github/workflows/premerge.yaml
@@ -70,12 +70,6 @@ jobs:
export SCCACHE_IDLE_TIMEOUT=0
sccache --start-server
- export projects_to_build=polly
- export project_check_targets=check-polly
- export runtimes_to_build=""
- export runtimes_check_targets=""
- export runtimes_check_targets_needs_reconfig=""
-
./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}" "${enable_cir}"
- name: Upload Artifacts
if: '!cancelled()'
@@ -112,9 +106,6 @@ jobs:
echo "Building projects: ${projects_to_build}"
echo "Running project checks targets: ${project_check_targets}"
- export projects_to_build=polly
- export project_check_targets=check-polly
-
echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT
echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT
- name: Build and Test
More information about the llvm-commits
mailing list