[clang] [llvm] [ci] Include a log download link when test report is truncated (PR #117985)
David Spickett via cfe-commits
cfe-commits at lists.llvm.org
Thu Nov 28 02:04:36 PST 2024
https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/117985
Now "Download" will be a link to the file so people don't have to know
to open the build tab and find the download button.
This is a URL from a real build:
https://buildkite.com/organizations/llvm-project/pipelines/github-pull-requests/builds/123979/jobs/01937132-0fc3-4c95-a884-2fc0048cb9a7/download.txt
And this is how we can build it:
https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/jobs/{BUILDKITE_JOB_ID}/download.txt
Given these env vars that were set in that job:
BUILDKITE_ORGANIZATION_SLUG="llvm-project"
BUILDKITE_PIPELINE_SLUG="github-pull-requests"
BUILDKITE_BUILD_NUMBER="123979"
BUILDKITE_JOB_ID="01937132-0fc3-4c95-a884-2fc0048cb9a7"
In theory these will always be available but:
1. Rather safe than sorry with this script, I don't want to make a passing
build a failure because this script failed.
2. It would get very annoying if you had to set all these to test
the script locally.
>From d05b35d5db09429441e777fbb7051bea60404163 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 28 Nov 2024 09:53:40 +0000
Subject: [PATCH 1/2] [ci] Include a log download link when test report is
truncated
Now "Download" will be a link to the file so people don't have to know
to open the build tab and find the download button.
This is a URL from a real build:
https://buildkite.com/organizations/llvm-project/pipelines/github-pull-requests/builds/123979/jobs/01937132-0fc3-4c95-a884-2fc0048cb9a7/download.txt
And this is how we can build it:
https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/jobs/{BUILDKITE_JOB_ID}/download.txt
Given these env vars that were set in that job:
BUILDKITE_ORGANIZATION_SLUG="llvm-project"
BUILDKITE_PIPELINE_SLUG="github-pull-requests"
BUILDKITE_BUILD_NUMBER="123979"
BUILDKITE_JOB_ID="01937132-0fc3-4c95-a884-2fc0048cb9a7"
In theory these will always be available but:
1. Rather safe than sorry with this script, I don't want to make a passing
build a failure because this script failed.
2. It would get very annoying if you had to set all these to test
the script locally.
---
.ci/generate_test_report.py | 88 ++++++++++++++++++++++++++++++++++---
1 file changed, 82 insertions(+), 6 deletions(-)
diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
index c44936b19dab98..6ba8ed432fad26 100644
--- a/.ci/generate_test_report.py
+++ b/.ci/generate_test_report.py
@@ -5,6 +5,7 @@
# python3 -m unittest discover -p generate_test_report.py
import argparse
+import os
import subprocess
import unittest
from io import StringIO
@@ -267,6 +268,46 @@ def test_report_dont_list_failures(self):
),
)
+ def test_report_dont_list_failures_link_to_log(self):
+ self.assertEqual(
+ _generate_report(
+ "Foo",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.02">
+ <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
+ <testcase classname="Bar/test_1" name="test_1" time="0.02">
+ <failure><![CDATA[Output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ list_failures=False,
+ buildkite_info={
+ "BUILDKITE_ORGANIZATION_SLUG": "organization_slug",
+ "BUILDKITE_PIPELINE_SLUG": "pipeline_slug",
+ "BUILDKITE_BUILD_NUMBER": "build_number",
+ "BUILDKITE_JOB_ID": "job_id",
+ },
+ ),
+ (
+ dedent(
+ """\
+ # Foo
+
+ * 1 test failed
+
+ Failed tests and their output was too large to report. [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details."""
+ ),
+ "error",
+ ),
+ )
+
def test_report_size_limit(self):
self.assertEqual(
_generate_report(
@@ -308,7 +349,13 @@ def test_report_size_limit(self):
# listed. This minimal report will always fit into an annotation.
# If include failures is False, total number of test will be reported but their names
# and output will not be.
-def _generate_report(title, junit_objects, size_limit=1024 * 1024, list_failures=True):
+def _generate_report(
+ title,
+ junit_objects,
+ size_limit=1024 * 1024,
+ list_failures=True,
+ buildkite_info=None,
+):
if not junit_objects:
return ("", "success")
@@ -354,11 +401,21 @@ def plural(num_tests):
report.append(f"* {tests_failed} {plural(tests_failed)} failed")
if not list_failures:
+ if buildkite_info is not None:
+ log_url = (
+ "https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/"
+ "pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/"
+ "jobs/{BUILDKITE_JOB_ID}/download.txt".format(**buildkite_info)
+ )
+ download_text = f"[Download]({log_url})"
+ else:
+ download_text = "Download"
+
report.extend(
[
"",
"Failed tests and their output was too large to report. "
- "Download the build's log file to see the details.",
+ f"{download_text} the build's log file to see the details.",
]
)
elif failures:
@@ -381,13 +438,21 @@ def plural(num_tests):
report = "\n".join(report)
if len(report.encode("utf-8")) > size_limit:
- return _generate_report(title, junit_objects, size_limit, list_failures=False)
+ return _generate_report(
+ title,
+ junit_objects,
+ size_limit,
+ list_failures=False,
+ buildkite_info=buildkite_info,
+ )
return report, style
-def generate_report(title, junit_files):
- return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files])
+def generate_report(title, junit_files, buildkite_info=None):
+ return _generate_report(
+ title, [JUnitXml.fromfile(p) for p in junit_files], buildkite_info
+ )
if __name__ == "__main__":
@@ -399,7 +464,18 @@ def generate_report(title, junit_files):
parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
args = parser.parse_args()
- report, style = generate_report(args.title, args.junit_files)
+ # All of these are required to build a link to download the log file.
+ env_var_names = [
+ "BUILDKITE_ORGANIZATION_SLUG",
+ "BUILDKITE_PIPELINE_SLUG",
+ "BUILDKITE_BUILD_NUMBER",
+ "BUILDKITE_JOB_ID",
+ ]
+ buildkite_info = {k: v for k, v in os.environ.items() if k in env_var_names}
+ if len(buildkite_info) != len(env_var_names):
+ buildkite_info = None
+
+ report, style = generate_report(args.title, args.junit_files, buildkite_info)
if report:
p = subprocess.Popen(
>From 3933eacf4519bf57b94341785debf048472d051c Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 28 Nov 2024 10:03:12 +0000
Subject: [PATCH 2/2] fail a whole bunch of tests
---
clang/tools/driver/driver.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index 12038de476ace1..174ad78c564dba 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -229,6 +229,9 @@ static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV,
}
int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContext) {
+ // Fail as many tests as possible!
+ return 1;
+
noteBottomOfStack();
llvm::setBugReportMsg("PLEASE submit a bug report to " BUG_REPORT_URL
" and include the crash backtrace, preprocessed "
More information about the cfe-commits
mailing list