[llvm] [CI] Handle Runtime Build Failures Correctly in generate_test_report (PR #155730)

[Aiden Grossman via llvm-commits]

https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/155730

>From bf7f89966ff3b306649f213ef3a2d34b546da362 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Thu, 28 Aug 2025 00:24:17 +0000
Subject: [PATCH 1/3] [CI] Handle Runtime Build Failures Correctly in
 generate_test_report

The nested ninja invocations currently confuse the script. Update  the
script to handle this correctly and a test to ensure we do not regress
this behavior in the future.
---
 .ci/generate_test_report_lib.py      |  7 +++++++
 .ci/generate_test_report_lib_test.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index d868c08ab69ef..5026c292a7934 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -27,6 +27,13 @@ def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
             # We hit the end of the log without finding a build failure, go to
             # the next log.
             return failures
+        # If we are doing a build with LLVM_ENABLE_RUNTIMES, we can have nested
+        # ninja invocations. The sub-ninja will print that a subcommand failed,
+        # and then the outer ninja will list the command that failed. We should
+        # ignore the outer failure.
+        if ninja_log[index - 1].startswith("ninja: build stopped:"):
+            index += 1
+            continue
         # We are trying to parse cases like the following:
         #
         # [4/5] test/4.stamp
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 466a8234776dc..96b21ffa2f88b 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -125,6 +125,34 @@ def test_ninja_log_multiple_failures(self):
                 ),
             ),
         )
+    
+    def test_ninja_log_runtimes_failure(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/5] test/1.stamp",
+                    "[2/5] test/2.stamp",
+                    "FAILED: touch test/2.stamp",
+                    "Wow! This system is really broken!",
+                    "ninja: build stopped: subcommand failed.",
+                    "FAILED: running check-runtime failed.",
+                    "<some random command>",
+                    "ninja: build stopped: subcommand failed.",
+                ]
+            ]
+        )
+        self.assertEqual(len(failures), 1)
+        self.assertEqual(
+            failures[0],
+            (
+                "test/2.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/2.stamp
+                    Wow! This system is really broken!"""
+                ),
+            ),
+        )
 
     def test_title_only(self):
         self.assertEqual(

>From 3b39ad52d0b46a1c81e7f93c60ec73c15a55dbfb Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Thu, 28 Aug 2025 00:26:25 +0000
Subject: [PATCH 2/3] formatting

---
 .ci/generate_test_report_lib_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 96b21ffa2f88b..d74bb55533c88 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -125,7 +125,7 @@ def test_ninja_log_multiple_failures(self):
                 ),
             ),
         )
-    
+
     def test_ninja_log_runtimes_failure(self):
         failures = generate_test_report_lib.find_failure_in_ninja_logs(
             [

>From 58612b6f62c8f723bdca482c681eafca6f45f0c1 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Thu, 28 Aug 2025 14:20:38 +0000
Subject: [PATCH 3/3] Add comment

---
 .ci/generate_test_report_lib_test.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index d74bb55533c88..a8659e1d6a3e3 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -126,6 +126,11 @@ def test_ninja_log_multiple_failures(self):
             ),
         )
 
+    # Test that we can correctly handle the runtimes build. the LLVM runtimes
+    # build will involve ninja invoking more ninja processes within the
+    # runtimes directory. This means that we see two failures for a failure in
+    # the runtimes build: one from the inner ninja containing the actual action
+    # that failed, and one for the sub ninja invocation that failed.
     def test_ninja_log_runtimes_failure(self):
         failures = generate_test_report_lib.find_failure_in_ninja_logs(
             [