[llvm] [CI] Add Support for Parsing Ninja Logs to generate_test_report_lib (PR #152620)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 7 18:47:13 PDT 2025
https://github.com/boomanaiden154 created https://github.com/llvm/llvm-project/pull/152620
This patch adds in support for taking the CLI output from ninja and
parsing it for failures. This is intended to be used in the cases where
all tests pass (or none have run), but the build fails to easily surface
where exactly the build failed.
The actual integration will happen in a future patch.
>From a33c37795668af455847b8d290964c0ccc6fb6ba Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 01:46:56 +0000
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
---
.ci/generate_test_report_lib.py | 51 +++++++++++++
.ci/generate_test_report_lib_test.py | 107 +++++++++++++++++++++++++++
2 files changed, 158 insertions(+)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 25d810f1c6d17..97b293507e3a2 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -12,6 +12,57 @@
"https://github.com/llvm/llvm-project/issues and add the "
"`infrastructure` label."
)
+# The maximum number of lines to pull from a ninja failure.
+NINJA_LOG_SIZE_THRESHOLD = 500
+
+
+def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
+ """Parses an individual ninja log."""
+ failures = []
+ index = 0
+ while index < len(ninja_log):
+ while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):
+ index += 1
+ if index == len(ninja_log):
+ # We hit the end of the log without finding a build failure, go to
+ # the next log.
+ return failures
+ failing_action = ninja_log[index - 1].split("] ")[1]
+ failure_log = []
+ while (
+ index < len(ninja_log)
+ and not ninja_log[index].startswith("[")
+ and not ninja_log[index].startswith(
+ "ninja: build stopped: subcommand failed"
+ )
+ and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
+ ):
+ failure_log.append(ninja_log[index])
+ index += 1
+ failures.append((failing_action, "\n".join(failure_log)))
+ return failures
+
+
+def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
+ """Extracts failure messages from ninja output.
+
+ This patch takes stdout/stderr from ninja in the form of a list of files
+ represented as a list of lines. This function then returns tuples containing
+ the name of the target and the error message.
+
+ Args:
+ ninja_logs: A list of files in the form of a list of lines representing the log
+ files captured from ninja.
+
+ Returns:
+ A list of tuples. The first string is the name of the target that failed. The
+ second string is the error message.
+ """
+ failures = []
+ for ninja_log in ninja_logs:
+ log_failures = _parse_ninja_log(ninja_log)
+ failures.extend(log_failures)
+ return failures
# Set size_limit to limit the byte size of the report. The default is 1MB as this
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index eda76ead19b9d..9b12c7d64514c 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -19,6 +19,113 @@ def junit_from_xml(xml):
class TestReports(unittest.TestCase):
+ def test_find_failure_ninja_logs(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: test/4.stamp",
+ "touch test/4.stamp",
+ "Wow! This system is really broken!",
+ "[5/5] test/5.stamp",
+ ],
+ ]
+ )
+ self.assertEqual(len(failures), 1)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/4.stamp",
+ dedent(
+ """\
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+
+ def test_no_failure_ninja_log(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/3] test/1.stamp",
+ "[2/3] test/2.stamp",
+ "[3/3] test/3.stamp",
+ ]
+ ]
+ )
+ self.assertEqual(failures, [])
+
+ def test_ninja_log_end(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/3] test/1.stamp",
+ "[2/3] test/2.stamp",
+ "[3/3] test/3.stamp",
+ "FAILED: touch test/3.stamp",
+ "Wow! This system is really broken!",
+ "ninja: build stopped: subcommand failed.",
+ ]
+ ]
+ )
+ self.assertEqual(len(failures), 1)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/3.stamp",
+ dedent(
+ """\
+ FAILED: touch test/3.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+
+ def test_ninja_log_multiple_failures(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! This system is really broken!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! This system is maybe broken!",
+ "[5/5] test/5.stamp",
+ ]
+ ]
+ )
+ self.assertEqual(len(failures), 2)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/2.stamp",
+ dedent(
+ """\
+ FAILED: touch test/2.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+ self.assertEqual(
+ failures[1],
+ (
+ "test/4.stamp",
+ dedent(
+ """\
+ FAILED: touch test/4.stamp
+ Wow! This system is maybe broken!"""
+ ),
+ ),
+ )
+
def test_title_only(self):
self.assertEqual(
generate_test_report_lib.generate_report("Foo", 0, []),
More information about the llvm-commits
mailing list