[llvm] [CI] Add Support for Parsing Ninja Logs to generate_test_report_lib (PR #152620)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 8 08:02:22 PDT 2025
https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/152620
>From a33c37795668af455847b8d290964c0ccc6fb6ba Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 01:46:56 +0000
Subject: [PATCH 1/3] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.6
---
.ci/generate_test_report_lib.py | 51 +++++++++++++
.ci/generate_test_report_lib_test.py | 107 +++++++++++++++++++++++++++
2 files changed, 158 insertions(+)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 25d810f1c6d17..97b293507e3a2 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -12,6 +12,57 @@
"https://github.com/llvm/llvm-project/issues and add the "
"`infrastructure` label."
)
+# The maximum number of lines to pull from a ninja failure.
+NINJA_LOG_SIZE_THRESHOLD = 500
+
+
+def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
+ """Parses an individual ninja log."""
+ failures = []
+ index = 0
+ while index < len(ninja_log):
+ while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):
+ index += 1
+ if index == len(ninja_log):
+ # We hit the end of the log without finding a build failure, go to
+ # the next log.
+ return failures
+ failing_action = ninja_log[index - 1].split("] ")[1]
+ failure_log = []
+ while (
+ index < len(ninja_log)
+ and not ninja_log[index].startswith("[")
+ and not ninja_log[index].startswith(
+ "ninja: build stopped: subcommand failed"
+ )
+ and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
+ ):
+ failure_log.append(ninja_log[index])
+ index += 1
+ failures.append((failing_action, "\n".join(failure_log)))
+ return failures
+
+
+def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
+ """Extracts failure messages from ninja output.
+
+ This patch takes stdout/stderr from ninja in the form of a list of files
+ represented as a list of lines. This function then returns tuples containing
+ the name of the target and the error message.
+
+ Args:
+ ninja_logs: A list of files in the form of a list of lines representing the log
+ files captured from ninja.
+
+ Returns:
+ A list of tuples. The first string is the name of the target that failed. The
+ second string is the error message.
+ """
+ failures = []
+ for ninja_log in ninja_logs:
+ log_failures = _parse_ninja_log(ninja_log)
+ failures.extend(log_failures)
+ return failures
# Set size_limit to limit the byte size of the report. The default is 1MB as this
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index eda76ead19b9d..9b12c7d64514c 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -19,6 +19,113 @@ def junit_from_xml(xml):
class TestReports(unittest.TestCase):
+ def test_find_failure_ninja_logs(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: test/4.stamp",
+ "touch test/4.stamp",
+ "Wow! This system is really broken!",
+ "[5/5] test/5.stamp",
+ ],
+ ]
+ )
+ self.assertEqual(len(failures), 1)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/4.stamp",
+ dedent(
+ """\
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+
+ def test_no_failure_ninja_log(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/3] test/1.stamp",
+ "[2/3] test/2.stamp",
+ "[3/3] test/3.stamp",
+ ]
+ ]
+ )
+ self.assertEqual(failures, [])
+
+ def test_ninja_log_end(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/3] test/1.stamp",
+ "[2/3] test/2.stamp",
+ "[3/3] test/3.stamp",
+ "FAILED: touch test/3.stamp",
+ "Wow! This system is really broken!",
+ "ninja: build stopped: subcommand failed.",
+ ]
+ ]
+ )
+ self.assertEqual(len(failures), 1)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/3.stamp",
+ dedent(
+ """\
+ FAILED: touch test/3.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+
+ def test_ninja_log_multiple_failures(self):
+ failures = generate_test_report_lib.find_failure_in_ninja_logs(
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! This system is really broken!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! This system is maybe broken!",
+ "[5/5] test/5.stamp",
+ ]
+ ]
+ )
+ self.assertEqual(len(failures), 2)
+ self.assertEqual(
+ failures[0],
+ (
+ "test/2.stamp",
+ dedent(
+ """\
+ FAILED: touch test/2.stamp
+ Wow! This system is really broken!"""
+ ),
+ ),
+ )
+ self.assertEqual(
+ failures[1],
+ (
+ "test/4.stamp",
+ dedent(
+ """\
+ FAILED: touch test/4.stamp
+ Wow! This system is maybe broken!"""
+ ),
+ ),
+ )
+
def test_title_only(self):
self.assertEqual(
generate_test_report_lib.generate_report("Foo", 0, []),
>From 9dfa1670f72049959b0a0dd9ff854040121057c0 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:01:58 +0000
Subject: [PATCH 2/3] feedback
Created using spr 1.3.6
---
.ci/generate_test_report_lib.py | 10 ++++++----
.ci/generate_test_report_lib_test.py | 6 ++----
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 97b293507e3a2..9b7af8c9314a3 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -27,14 +27,16 @@ def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
# We hit the end of the log without finding a build failure, go to
# the next log.
return failures
+ # index will point to the line that starts with Failed:. The progress
+ # indicator is the line before this and contains a pretty printed version
+ # of the target being built. We use this and remove the progress information
+ # to get a succinct name for the target.
failing_action = ninja_log[index - 1].split("] ")[1]
failure_log = []
while (
index < len(ninja_log)
and not ninja_log[index].startswith("[")
- and not ninja_log[index].startswith(
- "ninja: build stopped: subcommand failed"
- )
+ and not ninja_log[index].startswith("ninja: build stopped:")
and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
):
failure_log.append(ninja_log[index])
@@ -46,7 +48,7 @@ def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
"""Extracts failure messages from ninja output.
- This patch takes stdout/stderr from ninja in the form of a list of files
+ This function takes stdout/stderr from ninja in the form of a list of files
represented as a list of lines. This function then returns tuples containing
the name of the target and the error message.
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 9b12c7d64514c..41f3eae591e19 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -27,8 +27,7 @@ def test_find_failure_ninja_logs(self):
"[2/5] test/2.stamp",
"[3/5] test/3.stamp",
"[4/5] test/4.stamp",
- "FAILED: test/4.stamp",
- "touch test/4.stamp",
+ "FAILED: touch test/4.stamp",
"Wow! This system is really broken!",
"[5/5] test/5.stamp",
],
@@ -41,8 +40,7 @@ def test_find_failure_ninja_logs(self):
"test/4.stamp",
dedent(
"""\
- FAILED: test/4.stamp
- touch test/4.stamp
+ FAILED: touch test/4.stamp
Wow! This system is really broken!"""
),
),
>From 7103cbb7d8778f8fb459e64449620a69e43a4fd1 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 15:02:11 +0000
Subject: [PATCH 3/3] feedback
Created using spr 1.3.6
---
.ci/generate_test_report_lib.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 9b7af8c9314a3..df95db6a1d6b0 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -27,10 +27,16 @@ def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
# We hit the end of the log without finding a build failure, go to
# the next log.
return failures
+ # We are trying to parse cases like the following:
+ #
+ # [4/5] test/4.stamp
+ # FAILED: touch test/4.stamp
+ # touch test/4.stamp
+ #
# index will point to the line that starts with Failed:. The progress
- # indicator is the line before this and contains a pretty printed version
- # of the target being built. We use this and remove the progress information
- # to get a succinct name for the target.
+ # indicator is the line before this ([4/5] test/4.stamp) and contains a pretty
+ # printed version of the target being built (test/4.stamp). We use this line
+ # and remove the progress information to get a succinct name for the target.
failing_action = ninja_log[index - 1].split("] ")[1]
failure_log = []
while (
More information about the llvm-commits
mailing list