[llvm-branch-commits] [llvm] [CI] Setup generate_report to describe ninja failures (PR #152621)

[Aiden Grossman via llvm-branch-commits]

https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/152621

>From 53cd7281ec86ce8704faa8e4954c1fac1e3a7a8c Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:28:21 +0000
Subject: [PATCH 1/2] feedback

Created using spr 1.3.6
---
 .ci/generate_test_report_lib.py      | 8 ++++++++
 .ci/generate_test_report_lib_test.py | 3 +++
 2 files changed, 11 insertions(+)

diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index f62ab817fdb50..e4152ce54d352 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -219,6 +219,14 @@ def plural(num_tests):
                 ]
             )
         else:
+            report.extend(
+                [
+                    "",
+                    "All tests passed but another part of the build **failed**. Click on "
+                    "a failure below to see the details.",
+                    "",
+                ]
+            )
             report.extend(_format_ninja_failures(ninja_failures))
 
     if failures or return_code != 0:
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index e07875a4bd10d..e91ea48c704c6 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -323,6 +323,9 @@ def test_no_failures_build_failed_ninja_log(self):
                     # Foo
 
                     * 1 test passed
+
+                    All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
                     <details>
                     <summary>test/4.stamp</summary>
 

>From d70badb22b7aea2f934b551291e29609d5e9b540 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:52:52 +0000
Subject: [PATCH 2/2] feedback

Created using spr 1.3.6
---
 .ci/generate_test_report_lib_test.py | 94 ++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)

diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index e91ea48c704c6..389d781042e23 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -341,6 +341,100 @@ def test_no_failures_build_failed_ninja_log(self):
             ),
         )
 
+    def test_no_failures_multiple_build_failed_ninja_log(self):
+        test = generate_test_report_lib.generate_report(
+            "Foo",
+            1,
+            [
+                junit_from_xml(
+                    dedent(
+                        """\
+          <?xml version="1.0" encoding="UTF-8"?>
+          <testsuites time="0.00">
+          <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+          <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+          </testsuite>
+          </testsuites>"""
+                    )
+                )
+            ],
+            [
+                [
+                    "[1/5] test/1.stamp",
+                    "[2/5] test/2.stamp",
+                    "FAILED: touch test/2.stamp",
+                    "Wow! Be Kind!",
+                    "[3/5] test/3.stamp",
+                    "[4/5] test/4.stamp",
+                    "FAILED: touch test/4.stamp",
+                    "Wow! I Dare You!",
+                    "[5/5] test/5.stamp",
+                ]
+            ],
+        )
+        print(test)
+        self.assertEqual(
+            generate_test_report_lib.generate_report(
+                "Foo",
+                1,
+                [
+                    junit_from_xml(
+                        dedent(
+                            """\
+          <?xml version="1.0" encoding="UTF-8"?>
+          <testsuites time="0.00">
+          <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+          <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+          </testsuite>
+          </testsuites>"""
+                        )
+                    )
+                ],
+                [
+                    [
+                        "[1/5] test/1.stamp",
+                        "[2/5] test/2.stamp",
+                        "FAILED: touch test/2.stamp",
+                        "Wow! Be Kind!",
+                        "[3/5] test/3.stamp",
+                        "[4/5] test/4.stamp",
+                        "FAILED: touch test/4.stamp",
+                        "Wow! I Dare You!",
+                        "[5/5] test/5.stamp",
+                    ]
+                ],
+            ),
+            (
+                dedent(
+                    """\
+                    # Foo
+
+                    * 1 test passed
+
+                    All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+                    <details>
+                    <summary>test/2.stamp</summary>
+
+                    ```
+                    FAILED: touch test/2.stamp
+                    Wow! Be Kind!
+                    ```
+                    </details>
+                    <details>
+                    <summary>test/4.stamp</summary>
+
+                    ```
+                    FAILED: touch test/4.stamp
+                    Wow! I Dare You!
+                    ```
+                    </details>
+
+                    If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+                )
+            ),
+        )
+
     def test_report_single_file_single_testsuite(self):
         self.assertEqual(
             generate_test_report_lib.generate_report(