[clang] [llvm] Ci report script testing! (PR #113447)
David Spickett via cfe-commits
cfe-commits at lists.llvm.org
Wed Oct 23 04:37:11 PDT 2024
https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/113447
Do not commit!
>From ba99bed96bd69db7c86d43a2119255569872f41f Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Mon, 21 Oct 2024 12:34:17 +0000
Subject: [PATCH 1/4] [ci] Write test results to unique file names
In this patch I'm using a new lit option so that the pipeline
writes many results files, one for each time lit is run:
```
--use-unique-output-file-name
When enabled, lit will add a unique element to the output file name, before the extension. For example "results.xml" will become "results.<something>.xml". The
"<something>" is not ordered in any way and is chosen so that existing files are not overwritten. [Default: Off]
```
(I added this to lit recently)
Now if I run the Linux build:
$ bash ./.ci/monolithic-linux.sh "clang;lldb;lld" "check-lldb-shell check-lld" "libcxx;libcxxabi" "check-libcxx check-libcxxabi"
I get multiple test result files. In my case some tests fail so runtimes aren't checked, but all projects are
so there is 1 file for lldb and one for lld:
$ ls build/*.xml
build/test-results.klc82utf.xml build/test-results.majylh73.xml
This change just collects the XML files as artifacts. Once I know that's
working, I can setup a test reporting plugin to build a summary from them.
---
.ci/generate-buildkite-pipeline-premerge | 4 ++--
.ci/monolithic-linux.sh | 13 +++++++++----
.ci/monolithic-windows.sh | 2 +-
3 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 7676ff716c4185..e52133751f09b1 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -272,7 +272,7 @@ if [[ "${linux_projects}" != "" ]]; then
artifact_paths:
- 'artifacts/**/*'
- '*_result.json'
- - 'build/test-results.xml'
+ - 'build/test-results*.xml'
agents: ${LINUX_AGENTS}
retry:
automatic:
@@ -295,7 +295,7 @@ if [[ "${windows_projects}" != "" ]]; then
artifact_paths:
- 'artifacts/**/*'
- '*_result.json'
- - 'build/test-results.xml'
+ - 'build/test-results*.xml'
agents: ${WINDOWS_AGENTS}
retry:
automatic:
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index b78dc59432b65c..17ea51c08fafd3 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -37,6 +37,8 @@ trap show-stats EXIT
projects="${1}"
targets="${2}"
+lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
+
echo "--- cmake"
pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
@@ -47,7 +49,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
- -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
+ -D LLVM_LIT_ARGS="${lit_args}" \
-D LLVM_ENABLE_LLD=ON \
-D CMAKE_CXX_FLAGS=-gmlt \
-D LLVM_CCACHE_BUILD=ON \
@@ -87,7 +89,8 @@ if [[ "${runtimes}" != "" ]]; then
-D CMAKE_BUILD_TYPE=RelWithDebInfo \
-D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-D LIBCXX_TEST_PARAMS="std=c++03" \
- -D LIBCXXABI_TEST_PARAMS="std=c++03"
+ -D LIBCXXABI_TEST_PARAMS="std=c++03" \
+ -D LLVM_LIT_ARGS="${lit_args}"
echo "--- ninja runtimes C++03"
@@ -104,7 +107,8 @@ if [[ "${runtimes}" != "" ]]; then
-D CMAKE_BUILD_TYPE=RelWithDebInfo \
-D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-D LIBCXX_TEST_PARAMS="std=c++26" \
- -D LIBCXXABI_TEST_PARAMS="std=c++26"
+ -D LIBCXXABI_TEST_PARAMS="std=c++26" \
+ -D LLVM_LIT_ARGS="${lit_args}"
echo "--- ninja runtimes C++26"
@@ -121,7 +125,8 @@ if [[ "${runtimes}" != "" ]]; then
-D CMAKE_BUILD_TYPE=RelWithDebInfo \
-D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-D LIBCXX_TEST_PARAMS="enable_modules=clang" \
- -D LIBCXXABI_TEST_PARAMS="enable_modules=clang"
+ -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
+ -D LLVM_LIT_ARGS="${lit_args}"
echo "--- ninja runtimes clang modules"
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 91e719c52d4363..9ec44c22442d06 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -53,7 +53,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
- -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
+ -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests" \
-D COMPILER_RT_BUILD_ORC=OFF \
-D CMAKE_C_COMPILER_LAUNCHER=sccache \
-D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
>From c2d541afc37226a09eb89cf7b6e71836889940fd Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Wed, 23 Oct 2024 11:39:15 +0100
Subject: [PATCH 2/4] WIP
---
.ci/generate_test_report.py | 251 ++++++++++++++++++++++++++++++++++++
1 file changed, 251 insertions(+)
create mode 100644 .ci/generate_test_report.py
diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
new file mode 100644
index 00000000000000..672b131dbbd311
--- /dev/null
+++ b/.ci/generate_test_report.py
@@ -0,0 +1,251 @@
+# Script to parse many JUnit XML result files and produce a combined report
+# that CI jobs can show.
+#
+# To run the unittests:
+# python3 -m unittest discover -p generate_test_report.py
+
+import argparse
+import unittest
+from io import StringIO
+from junitparser import JUnitXml, Failure
+from textwrap import dedent
+
+
+def junit_from_xml(xml):
+ return JUnitXml.fromfile(StringIO(xml))
+
+
+class TestReports(unittest.TestCase):
+ def test_title(self):
+ self.assertEqual(_generate_report("Foo", []), "# Foo")
+
+ def test_report_single_file_single_testsuite(self):
+ self.assertEqual(
+ _generate_report(
+ "Foo",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="8.89">
+ <testsuite name="Bar" tests="4" failures="2" skipped="1" time="410.63">
+ <testcase classname="Bar/test_1" name="test_1" time="0.02"/>
+ <testcase classname="Bar/test_2" name="test_2" time="0.02">
+ <skipped message="Reason"/>
+ </testcase>
+ <testcase classname="Bar/test_3" name="test_3" time="0.02">
+ <failure><![CDATA[Output goes here]]></failure>
+ </testcase>
+ <testcase classname="Bar/test_4" name="test_4" time="0.02">
+ <failure><![CDATA[Other output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ ),
+ dedent(
+ """\
+ # Foo
+
+ 1 tests passed
+ 1 tests skipped
+ 2 tests failed
+
+ ## Failed tests
+ (click to see output)
+
+ ### Bar
+ <details>
+ <summary>Bar/test_3/test_3</summary>
+
+ ```
+ Output goes here
+ ```
+ </details>
+ <details>
+ <summary>Bar/test_4/test_4</summary>
+
+ ```
+ Other output goes here
+ ```
+ </details>"""
+ ),
+ )
+
+ MULTI_SUITE_OUTPUT = dedent(
+ """\
+ # ABC and DEF
+
+ 1 tests passed
+ 1 tests skipped
+ 2 tests failed
+
+ ## Failed tests
+ (click to see output)
+
+ ### ABC
+ <details>
+ <summary>ABC/test_2/test_2</summary>
+
+ ```
+ ABC/test_2 output goes here
+ ```
+ </details>
+
+ ### DEF
+ <details>
+ <summary>DEF/test_2/test_2</summary>
+
+ ```
+ DEF/test_2 output goes here
+ ```
+ </details>"""
+ )
+
+ def test_report_single_file_multiple_testsuites(self):
+ self.assertEqual(
+ _generate_report(
+ "ABC and DEF",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="8.89">
+ <testsuite name="ABC" tests="2" failures="1" skipped="0" time="410.63">
+ <testcase classname="ABC/test_1" name="test_1" time="0.02"/>
+ <testcase classname="ABC/test_2" name="test_2" time="0.02">
+ <failure><![CDATA[ABC/test_2 output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ <testsuite name="DEF" tests="2" failures="1" skipped="1" time="410.63">
+ <testcase classname="DEF/test_1" name="test_1" time="0.02">
+ <skipped message="reason"/>
+ </testcase>
+ <testcase classname="DEF/test_2" name="test_2" time="0.02">
+ <failure><![CDATA[DEF/test_2 output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ ),
+ self.MULTI_SUITE_OUTPUT,
+ )
+
+ def test_report_multiple_files_multiple_testsuites(self):
+ self.assertEqual(
+ _generate_report(
+ "ABC and DEF",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="8.89">
+ <testsuite name="ABC" tests="2" failures="1" skipped="0" time="410.63">
+ <testcase classname="ABC/test_1" name="test_1" time="0.02"/>
+ <testcase classname="ABC/test_2" name="test_2" time="0.02">
+ <failure><![CDATA[ABC/test_2 output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ ),
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="8.89">
+ <testsuite name="DEF" tests="2" failures="1" skipped="1" time="410.63">
+ <testcase classname="DEF/test_1" name="test_1" time="0.02">
+ <skipped message="reason"/>
+ </testcase>
+ <testcase classname="DEF/test_2" name="test_2" time="0.02">
+ <failure><![CDATA[DEF/test_2 output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ ),
+ ],
+ ),
+ self.MULTI_SUITE_OUTPUT,
+ )
+
+
+def _generate_report(title, junit_objects):
+ report = [f"# {title}"]
+
+ if junit_objects:
+ report.append("")
+ failures = {}
+
+ tests_run = 0
+ tests_skipped = 0
+ tests_failed = 0
+
+ for results in junit_objects:
+ for testsuite in results:
+ tests_run += testsuite.tests
+ tests_skipped += testsuite.skipped
+ tests_failed += testsuite.failures
+
+ for test in testsuite:
+ if (
+ not test.is_passed
+ and test.result
+ and isinstance(test.result[0], Failure)
+ ):
+ if failures.get(testsuite.name) is None:
+ failures[testsuite.name] = []
+ failures[testsuite.name].append(
+ (test.classname + "/" + test.name, test.result[0].text)
+ )
+
+ tests_passed = tests_run - tests_skipped - tests_failed
+ report.extend(
+ [
+ f"{tests_passed} tests passed",
+ f"{tests_skipped} tests skipped",
+ f"{tests_failed} tests failed",
+ ]
+ )
+
+ if failures:
+ report.extend(["", "## Failed tests", "(click to see output)"])
+ for testsuite_name, failures in failures.items():
+ report.extend(["", f"### {testsuite_name}"])
+ for name, output in failures:
+ report.extend(
+ [
+ "<details>",
+ f"<summary>{name}</summary>",
+ "",
+ "```",
+ output,
+ "```",
+ "</details>",
+ ]
+ )
+
+ return "\n".join(report)
+
+
+def generate_report(title, junit_files):
+ return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files])
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "title", help="Title of the test report, without Markdown formatting."
+ )
+ parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
+ args = parser.parse_args()
+
+ print(generate_report(args.title, args.junit_files))
>From dc897ccd87ce96aca24f116c46f25e1a3bfc0d1d Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Wed, 23 Oct 2024 12:30:07 +0100
Subject: [PATCH 3/4] Add deliberate test failures
---
clang/test/PCH/arc.m | 2 +-
llvm/test/MC/AArch64/adr.s | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/test/PCH/arc.m b/clang/test/PCH/arc.m
index e4ad71a469b956..ec7a4a2249ec23 100644
--- a/clang/test/PCH/arc.m
+++ b/clang/test/PCH/arc.m
@@ -14,5 +14,5 @@
array0 a0;
array1 a1;
-// CHECK-ERR1: Objective-C automated reference counting was enabled in AST file '{{.*}}' but is currently disabled
+// CHECK-ERR1: Objective-C automated reference counting was enabled in AST file '{{.*}}' but is currently disabled and this will fail the test
// CHECK-ERR2: Objective-C automated reference counting was disabled in AST file '{{.*}}' but is currently enabled
diff --git a/llvm/test/MC/AArch64/adr.s b/llvm/test/MC/AArch64/adr.s
index c2ddc8fcb38b56..ef9491209a01c9 100644
--- a/llvm/test/MC/AArch64/adr.s
+++ b/llvm/test/MC/AArch64/adr.s
@@ -19,7 +19,7 @@
adr x5, (0xffffffff000f1000 - 0xffffffff00000000 + Symbol)
adr x6, Symbol + (0xffffffff000f1000 - 0xffffffff00000000)
-// CHECK-NEXT: adrp x0, 0x0
+// CHECK-NEXT: adrp x0, 0x0 -- will fail the test!!!
// CHECK-NEXT: R_AARCH64_ADR_PREL_PG_HI21 Symbol
// CHECK-NEXT: adrp x2, 0x0
// CHECK-NEXT: R_AARCH64_ADR_PREL_PG_HI21 Symbol
>From 0c8176c4f595b174db68fedc2b2ae110813554ce Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Wed, 23 Oct 2024 12:36:31 +0100
Subject: [PATCH 4/4] hook this up to the linux build
---
.ci/generate-buildkite-pipeline-premerge | 48 ++++++++++++------------
.ci/monolithic-linux.sh | 8 +++-
2 files changed, 30 insertions(+), 26 deletions(-)
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index e52133751f09b1..06defbb8d037f5 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -289,27 +289,27 @@ if [[ "${linux_projects}" != "" ]]; then
EOF
fi
-if [[ "${windows_projects}" != "" ]]; then
- cat <<EOF
-- label: ':windows: Windows x64'
- artifact_paths:
- - 'artifacts/**/*'
- - '*_result.json'
- - 'build/test-results*.xml'
- agents: ${WINDOWS_AGENTS}
- retry:
- automatic:
- - exit_status: -1 # Agent was lost
- limit: 2
- - exit_status: 255 # Forced agent shutdown
- limit: 2
- timeout_in_minutes: 150
- env:
- CC: 'cl'
- CXX: 'cl'
- LD: 'link'
- commands:
- - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
- - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
-EOF
-fi
+# if [[ "${windows_projects}" != "" ]]; then
+# cat <<EOF
+# - label: ':windows: Windows x64'
+# artifact_paths:
+# - 'artifacts/**/*'
+# - '*_result.json'
+# - 'build/test-results*.xml'
+# agents: ${WINDOWS_AGENTS}
+# retry:
+# automatic:
+# - exit_status: -1 # Agent was lost
+# limit: 2
+# - exit_status: 255 # Forced agent shutdown
+# limit: 2
+# timeout_in_minutes: 150
+# env:
+# CC: 'cl'
+# CXX: 'cl'
+# LD: 'link'
+# commands:
+# - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
+# - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
+# EOF
+# fi
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 17ea51c08fafd3..965039b28e6af4 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -28,11 +28,14 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
ccache --clear
fi
-function show-stats {
+function at-exit {
+ python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64 Test Results" "${BUILD_DIR}"/test-results*.xml | \
+ buildkite-agent annotate --context "linux-x64-test-results"
+
mkdir -p artifacts
ccache --print-stats > artifacts/ccache_stats.txt
}
-trap show-stats EXIT
+trap at-exit EXIT
projects="${1}"
targets="${2}"
@@ -42,6 +45,7 @@ lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-outpu
echo "--- cmake"
pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
+pip install -q junitparser==3.2.0
cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_PROJECTS="${projects}" \
-G Ninja \
More information about the cfe-commits
mailing list