[llvm-branch-commits] [llvm] [CI] Setup generate_report to describe ninja failures (PR #152621)
Aiden Grossman via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 8 08:51:39 PDT 2025
https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/152621
>From 53cd7281ec86ce8704faa8e4954c1fac1e3a7a8c Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:28:21 +0000
Subject: [PATCH 1/4] feedback
Created using spr 1.3.6
---
.ci/generate_test_report_lib.py | 8 ++++++++
.ci/generate_test_report_lib_test.py | 3 +++
2 files changed, 11 insertions(+)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index f62ab817fdb50..e4152ce54d352 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -219,6 +219,14 @@ def plural(num_tests):
]
)
else:
+ report.extend(
+ [
+ "",
+ "All tests passed but another part of the build **failed**. Click on "
+ "a failure below to see the details.",
+ "",
+ ]
+ )
report.extend(_format_ninja_failures(ninja_failures))
if failures or return_code != 0:
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index e07875a4bd10d..e91ea48c704c6 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -323,6 +323,9 @@ def test_no_failures_build_failed_ninja_log(self):
# Foo
* 1 test passed
+
+ All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
<details>
<summary>test/4.stamp</summary>
>From d70badb22b7aea2f934b551291e29609d5e9b540 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 14:52:52 +0000
Subject: [PATCH 2/4] feedback
Created using spr 1.3.6
---
.ci/generate_test_report_lib_test.py | 94 ++++++++++++++++++++++++++++
1 file changed, 94 insertions(+)
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index e91ea48c704c6..389d781042e23 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -341,6 +341,100 @@ def test_no_failures_build_failed_ninja_log(self):
),
)
+ def test_no_failures_multiple_build_failed_ninja_log(self):
+ test = generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.00">
+ <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+ <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! Be Kind!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! I Dare You!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ )
+ print(test)
+ self.assertEqual(
+ generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.00">
+ <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+ <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! Be Kind!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! I Dare You!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ ),
+ (
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed
+
+ All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+ <details>
+ <summary>test/2.stamp</summary>
+
+ ```
+ FAILED: touch test/2.stamp
+ Wow! Be Kind!
+ ```
+ </details>
+ <details>
+ <summary>test/4.stamp</summary>
+
+ ```
+ FAILED: touch test/4.stamp
+ Wow! I Dare You!
+ ```
+ </details>
+
+ If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+ )
+ ),
+ )
+
def test_report_single_file_single_testsuite(self):
self.assertEqual(
generate_test_report_lib.generate_report(
>From d0e5bcea9cf22468b5dfff14cb88d955ac25d518 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 15:40:46 +0000
Subject: [PATCH 3/4] fix
Created using spr 1.3.6
---
.ci/generate_test_report_github.py | 6 ++-
.ci/generate_test_report_lib.py | 18 +++++++--
.ci/generate_test_report_lib_test.py | 58 ++++++++++++++++++++++++++++
.ci/utils.sh | 3 +-
4 files changed, 78 insertions(+), 7 deletions(-)
diff --git a/.ci/generate_test_report_github.py b/.ci/generate_test_report_github.py
index 4b7f3a2367d50..7242264723cbf 100644
--- a/.ci/generate_test_report_github.py
+++ b/.ci/generate_test_report_github.py
@@ -16,11 +16,13 @@
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("return_code", help="The build's return code.", type=int)
- parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
+ parser.add_argument(
+ "build_test_logs", help="Paths to JUnit report files and ninja logs.", nargs="*"
+ )
args = parser.parse_args()
report = generate_test_report_lib.generate_report_from_files(
- PLATFORM_TITLES[platform.system()], args.return_code, args.junit_files
+ PLATFORM_TITLES[platform.system()], args.return_code, args.build_test_logs
)
print(report)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 419913325a156..d868c08ab69ef 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -251,9 +251,19 @@ def plural(num_tests):
return report
-def generate_report_from_files(title, return_code, junit_files):
+def generate_report_from_files(title, return_code, build_log_files):
+ junit_files = [
+ junit_file for junit_file in build_log_files if junit_file.endswith(".xml")
+ ]
+ ninja_log_files = [
+ ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log")
+ ]
+ ninja_logs = []
+ for ninja_log_file in ninja_log_files:
+ with open(ninja_log_file, "r") as ninja_log_file_handle:
+ ninja_logs.append(
+ [log_line.strip() for log_line in ninja_log_file_handle.readlines()]
+ )
return generate_report(
- title,
- return_code,
- [JUnitXml.fromfile(p) for p in junit_files],
+ title, return_code, [JUnitXml.fromfile(p) for p in junit_files], ninja_logs
)
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 389d781042e23..466a8234776dc 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -8,6 +8,8 @@
import unittest
from io import StringIO
from textwrap import dedent
+import tempfile
+import os
from junitparser import JUnitXml
@@ -718,3 +720,59 @@ def test_report_size_limit(self):
)
),
)
+
+ def test_generate_report_end_to_end(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ junit_xml_file = os.path.join(temp_dir, "junit.xml")
+ with open(junit_xml_file, "w") as junit_xml_handle:
+ junit_xml_handle.write(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.00">
+ <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+ <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ninja_log_file = os.path.join(temp_dir, "ninja.log")
+ with open(ninja_log_file, "w") as ninja_log_handle:
+ ninja_log_handle.write(
+ dedent(
+ """\
+ [1/5] test/1.stamp
+ [2/5] test/2.stamp
+ [3/5] test/3.stamp
+ [4/5] test/4.stamp
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! That's so True!
+ [5/5] test/5.stamp"""
+ )
+ )
+ self.assertEqual(
+ generate_test_report_lib.generate_report_from_files(
+ "Foo", 1, [junit_xml_file, ninja_log_file]
+ ),
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed
+
+ All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+ <details>
+ <summary>test/4.stamp</summary>
+
+ ```
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! That's so True!
+ ```
+ </details>
+
+ If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+ ),
+ )
diff --git a/.ci/utils.sh b/.ci/utils.sh
index 6656ffe06666a..30bf2d9df5ca1 100644
--- a/.ci/utils.sh
+++ b/.ci/utils.sh
@@ -33,7 +33,8 @@ function at-exit {
if [[ "$GITHUB_STEP_SUMMARY" != "" ]]; then
python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py \
- $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
+ $retcode "${BUILD_DIR}"/test-results.*.xml "${BUILD_DIR}"/ninja*.log \
+ >> $GITHUB_STEP_SUMMARY
fi
}
trap at-exit EXIT
>From 7cab5811a3aa7d24120ebd7dbd8d555c6383ab0e Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Fri, 8 Aug 2025 15:51:28 +0000
Subject: [PATCH 4/4] fix
Created using spr 1.3.6
---
.ci/generate_test_report_github.py | 6 +--
.ci/generate_test_report_lib.py | 18 ++-------
.ci/generate_test_report_lib_test.py | 58 ----------------------------
.ci/utils.sh | 3 +-
4 files changed, 7 insertions(+), 78 deletions(-)
diff --git a/.ci/generate_test_report_github.py b/.ci/generate_test_report_github.py
index 7242264723cbf..4b7f3a2367d50 100644
--- a/.ci/generate_test_report_github.py
+++ b/.ci/generate_test_report_github.py
@@ -16,13 +16,11 @@
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("return_code", help="The build's return code.", type=int)
- parser.add_argument(
- "build_test_logs", help="Paths to JUnit report files and ninja logs.", nargs="*"
- )
+ parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
args = parser.parse_args()
report = generate_test_report_lib.generate_report_from_files(
- PLATFORM_TITLES[platform.system()], args.return_code, args.build_test_logs
+ PLATFORM_TITLES[platform.system()], args.return_code, args.junit_files
)
print(report)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index d868c08ab69ef..419913325a156 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -251,19 +251,9 @@ def plural(num_tests):
return report
-def generate_report_from_files(title, return_code, build_log_files):
- junit_files = [
- junit_file for junit_file in build_log_files if junit_file.endswith(".xml")
- ]
- ninja_log_files = [
- ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log")
- ]
- ninja_logs = []
- for ninja_log_file in ninja_log_files:
- with open(ninja_log_file, "r") as ninja_log_file_handle:
- ninja_logs.append(
- [log_line.strip() for log_line in ninja_log_file_handle.readlines()]
- )
+def generate_report_from_files(title, return_code, junit_files):
return generate_report(
- title, return_code, [JUnitXml.fromfile(p) for p in junit_files], ninja_logs
+ title,
+ return_code,
+ [JUnitXml.fromfile(p) for p in junit_files],
)
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 466a8234776dc..389d781042e23 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -8,8 +8,6 @@
import unittest
from io import StringIO
from textwrap import dedent
-import tempfile
-import os
from junitparser import JUnitXml
@@ -720,59 +718,3 @@ def test_report_size_limit(self):
)
),
)
-
- def test_generate_report_end_to_end(self):
- with tempfile.TemporaryDirectory() as temp_dir:
- junit_xml_file = os.path.join(temp_dir, "junit.xml")
- with open(junit_xml_file, "w") as junit_xml_handle:
- junit_xml_handle.write(
- dedent(
- """\
- <?xml version="1.0" encoding="UTF-8"?>
- <testsuites time="0.00">
- <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
- <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
- </testsuite>
- </testsuites>"""
- )
- )
- ninja_log_file = os.path.join(temp_dir, "ninja.log")
- with open(ninja_log_file, "w") as ninja_log_handle:
- ninja_log_handle.write(
- dedent(
- """\
- [1/5] test/1.stamp
- [2/5] test/2.stamp
- [3/5] test/3.stamp
- [4/5] test/4.stamp
- FAILED: test/4.stamp
- touch test/4.stamp
- Wow! That's so True!
- [5/5] test/5.stamp"""
- )
- )
- self.assertEqual(
- generate_test_report_lib.generate_report_from_files(
- "Foo", 1, [junit_xml_file, ninja_log_file]
- ),
- dedent(
- """\
- # Foo
-
- * 1 test passed
-
- All tests passed but another part of the build **failed**. Click on a failure below to see the details.
-
- <details>
- <summary>test/4.stamp</summary>
-
- ```
- FAILED: test/4.stamp
- touch test/4.stamp
- Wow! That's so True!
- ```
- </details>
-
- If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
- ),
- )
diff --git a/.ci/utils.sh b/.ci/utils.sh
index 30bf2d9df5ca1..6656ffe06666a 100644
--- a/.ci/utils.sh
+++ b/.ci/utils.sh
@@ -33,8 +33,7 @@ function at-exit {
if [[ "$GITHUB_STEP_SUMMARY" != "" ]]; then
python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py \
- $retcode "${BUILD_DIR}"/test-results.*.xml "${BUILD_DIR}"/ninja*.log \
- >> $GITHUB_STEP_SUMMARY
+ $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
fi
}
trap at-exit EXIT
More information about the llvm-branch-commits
mailing list