[clang] [llvm] Ci plugin (PR #113290)

David Spickett via cfe-commits cfe-commits at lists.llvm.org
Tue Oct 22 07:30:40 PDT 2024


https://github.com/DavidSpickett updated https://github.com/llvm/llvm-project/pull/113290

>From df728276026481b7397d2008541bfaed572774a7 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Mon, 21 Oct 2024 12:34:17 +0000
Subject: [PATCH 1/8] [ci] Write test results to unique file names

In this patch I'm using a new lit option so that the pipeline
writes many results files, one for each time lit is run:
```
--use-unique-output-file-name
  When enabled, lit will add a unique element to the output file name, before the extension. For example "results.xml" will become "results.<something>.xml". The
  "<something>" is not ordered in any way and is chosen so that existing files are not overwritten. [Default: Off]
```

(I added this to lit recently)

Now if I run the Linux build:
$ bash ./.ci/monolithic-linux.sh "clang;lldb;lld" "check-lldb-shell check-lld" "libcxx;libcxxabi" "check-libcxx check-libcxxabi"

I get multiple test result files. In my case some tests fail so runtimes aren't checked, but all projects are
so there is 1 file for lldb and one for lld:
$ ls build/*.xml
build/test-results.klc82utf.xml  build/test-results.majylh73.xml

This change just collects the XML files as artifacts. Once I know that's
working, I can setup a test reporting plugin to build a summary from them.
---
 .ci/generate-buildkite-pipeline-premerge |  4 ++--
 .ci/monolithic-linux.sh                  | 13 +++++++++----
 .ci/monolithic-windows.sh                |  2 +-
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 7676ff716c4185..e52133751f09b1 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -272,7 +272,7 @@ if [[ "${linux_projects}" != "" ]]; then
   artifact_paths:
   - 'artifacts/**/*'
   - '*_result.json'
-  - 'build/test-results.xml'
+  - 'build/test-results*.xml'
   agents: ${LINUX_AGENTS}
   retry:
     automatic:
@@ -295,7 +295,7 @@ if [[ "${windows_projects}" != "" ]]; then
   artifact_paths:
   - 'artifacts/**/*'
   - '*_result.json'
-  - 'build/test-results.xml'
+  - 'build/test-results*.xml'
   agents: ${WINDOWS_AGENTS}
   retry:
     automatic:
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index b78dc59432b65c..17ea51c08fafd3 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -37,6 +37,8 @@ trap show-stats EXIT
 projects="${1}"
 targets="${2}"
 
+lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
+
 echo "--- cmake"
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
@@ -47,7 +49,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
       -D COMPILER_RT_BUILD_LIBFUZZER=OFF \
-      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
+      -D LLVM_LIT_ARGS="${lit_args}" \
       -D LLVM_ENABLE_LLD=ON \
       -D CMAKE_CXX_FLAGS=-gmlt \
       -D LLVM_CCACHE_BUILD=ON \
@@ -87,7 +89,8 @@ if [[ "${runtimes}" != "" ]]; then
       -D CMAKE_BUILD_TYPE=RelWithDebInfo \
       -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
       -D LIBCXX_TEST_PARAMS="std=c++03" \
-      -D LIBCXXABI_TEST_PARAMS="std=c++03"
+      -D LIBCXXABI_TEST_PARAMS="std=c++03" \
+      -D LLVM_LIT_ARGS="${lit_args}"
 
   echo "--- ninja runtimes C++03"
 
@@ -104,7 +107,8 @@ if [[ "${runtimes}" != "" ]]; then
       -D CMAKE_BUILD_TYPE=RelWithDebInfo \
       -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
       -D LIBCXX_TEST_PARAMS="std=c++26" \
-      -D LIBCXXABI_TEST_PARAMS="std=c++26"
+      -D LIBCXXABI_TEST_PARAMS="std=c++26" \
+      -D LLVM_LIT_ARGS="${lit_args}"
 
   echo "--- ninja runtimes C++26"
 
@@ -121,7 +125,8 @@ if [[ "${runtimes}" != "" ]]; then
       -D CMAKE_BUILD_TYPE=RelWithDebInfo \
       -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
       -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
-      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang"
+      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
+      -D LLVM_LIT_ARGS="${lit_args}"
 
   echo "--- ninja runtimes clang modules"
   
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 91e719c52d4363..9ec44c22442d06 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -53,7 +53,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
       -D COMPILER_RT_BUILD_LIBFUZZER=OFF \
-      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
+      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests" \
       -D COMPILER_RT_BUILD_ORC=OFF \
       -D CMAKE_C_COMPILER_LAUNCHER=sccache \
       -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \

>From 3cee8894586e063848ce0ce58d1cbc668f21ce37 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Mon, 21 Oct 2024 13:13:48 +0000
Subject: [PATCH 2/8] add small changes to trigger a demo CI run

---
 clang/README.md | 2 ++
 llvm/README.txt | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/clang/README.md b/clang/README.md
index b98182d8a3f684..7cc2136be13488 100644
--- a/clang/README.md
+++ b/clang/README.md
@@ -23,3 +23,5 @@ If you're interested in more (including how to build Clang) it is best to read t
 * If you find a bug in Clang, please file it in the LLVM bug tracker:
   
     https://github.com/llvm/llvm-project/issues
+
+Test line.
\ No newline at end of file
diff --git a/llvm/README.txt b/llvm/README.txt
index b9b71a3b6daff1..d5e0796876bd13 100644
--- a/llvm/README.txt
+++ b/llvm/README.txt
@@ -15,3 +15,5 @@ documentation setup.
 
 If you are writing a package for LLVM, see docs/Packaging.rst for our
 suggestions.
+
+Test line.
\ No newline at end of file

>From 73de69eb9c318c66ae3cac5abaf87024e7ca1aa4 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Tue, 22 Oct 2024 09:53:36 +0000
Subject: [PATCH 3/8] WIP testing reporting plugin

---
 .ci/generate-buildkite-pipeline-premerge | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index e52133751f09b1..22576de9423eec 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -286,6 +286,11 @@ if [[ "${linux_projects}" != "" ]]; then
     CXX: 'clang++'
   commands:
   - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'
+  - wait: ~
+    continue_on_failure: true
+  - plugins:
+      - junit-annotate#v2.5.0:
+          artifacts: build/test-results*.xml
 EOF
 fi
 

>From 0d92f6fd37c394367b5f92e5d11d022d98cd97cc Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Tue, 22 Oct 2024 09:56:49 +0000
Subject: [PATCH 4/8] report windows too

---
 .ci/generate-buildkite-pipeline-premerge | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 22576de9423eec..6816094580a8ea 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -316,5 +316,10 @@ if [[ "${windows_projects}" != "" ]]; then
   commands:
   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
+  - wait: ~
+    continue_on_failure: true
+  - plugins:
+      - junit-annotate#v2.5.0:
+          artifacts: build/test-results*.xml
 EOF
 fi

>From c1a8fa22524228ca82312ee3ef7380ef73d30159 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Tue, 22 Oct 2024 11:41:16 +0000
Subject: [PATCH 5/8] report all steps?

---
 .ci/generate-buildkite-pipeline-premerge | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 6816094580a8ea..bc301a67fbf68d 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -286,11 +286,6 @@ if [[ "${linux_projects}" != "" ]]; then
     CXX: 'clang++'
   commands:
   - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'
-  - wait: ~
-    continue_on_failure: true
-  - plugins:
-      - junit-annotate#v2.5.0:
-          artifacts: build/test-results*.xml
 EOF
 fi
 
@@ -316,10 +311,13 @@ if [[ "${windows_projects}" != "" ]]; then
   commands:
   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
-  - wait: ~
-    continue_on_failure: true
-  - plugins:
-      - junit-annotate#v2.5.0:
-          artifacts: build/test-results*.xml
 EOF
 fi
+
+cat <<EOF
+- wait: ~
+  continue_on_failure: true
+- plugins:
+    - junit-annotate#v2.5.0:
+        artifacts: build/test-results*.xml
+EOF

>From b8d02da23a92935436ecbd38411bf9f4244e7cd9 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Tue, 22 Oct 2024 13:24:23 +0000
Subject: [PATCH 6/8] annotate per platform

---
 .ci/generate-buildkite-pipeline-premerge | 24 +++++++++++++++++-------
 .ci/monolithic-linux.sh                  |  2 +-
 .ci/monolithic-windows.sh                |  2 +-
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index bc301a67fbf68d..1a7fe7abbd7a79 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -269,10 +269,11 @@ windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 if [[ "${linux_projects}" != "" ]]; then
   cat <<EOF
 - label: ':linux: Linux x64'
+  key: "test Linux"
   artifact_paths:
   - 'artifacts/**/*'
   - '*_result.json'
-  - 'build/test-results*.xml'
+  - 'build/test-results-linux*.xml'
   agents: ${LINUX_AGENTS}
   retry:
     automatic:
@@ -286,16 +287,23 @@ if [[ "${linux_projects}" != "" ]]; then
     CXX: 'clang++'
   commands:
   - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'
+- plugins:
+    - junit-annotate#v2.5.0:
+        artifacts: build/test-results-linux*.xml
+        context: "linux-test-results"
+  depends_on: "test Linux"
+  allow_dependency_failure: true
 EOF
 fi
 
 if [[ "${windows_projects}" != "" ]]; then
   cat <<EOF
 - label: ':windows: Windows x64'
+  key: "test Windows"
   artifact_paths:
   - 'artifacts/**/*'
   - '*_result.json'
-  - 'build/test-results*.xml'
+  - 'build/test-results-windows*.xml'
   agents: ${WINDOWS_AGENTS}
   retry:
     automatic:
@@ -311,13 +319,15 @@ if [[ "${windows_projects}" != "" ]]; then
   commands:
   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
+- plugins:
+    - junit-annotate#v2.5.0:
+        artifacts: build/test-results-windows*.xml
+        context: "windows-test-results"
+  depends_on: "test Windows"
+  allow_dependency_failure: true
 EOF
 fi
 
 cat <<EOF
-- wait: ~
-  continue_on_failure: true
-- plugins:
-    - junit-annotate#v2.5.0:
-        artifacts: build/test-results*.xml
+
 EOF
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 17ea51c08fafd3..660c11764f7bdc 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -37,7 +37,7 @@ trap show-stats EXIT
 projects="${1}"
 targets="${2}"
 
-lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
+lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results-linux.xml --use-unique-output-file-name --timeout=1200 --time-tests"
 
 echo "--- cmake"
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 9ec44c22442d06..cde4df3a3ea83a 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -53,7 +53,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
       -D COMPILER_RT_BUILD_LIBFUZZER=OFF \
-      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests" \
+      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results-windows.xml --use-unique-output-file-name --timeout=1200 --time-tests" \
       -D COMPILER_RT_BUILD_ORC=OFF \
       -D CMAKE_C_COMPILER_LAUNCHER=sccache \
       -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \

>From 4d5b57cbc562c1e1afaa49ce68ab9c0f4ede0596 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Tue, 22 Oct 2024 13:25:18 +0000
Subject: [PATCH 7/8] Add some test failures to report

---
 clang/test/Sema/asm.c                             | 2 +-
 llvm/test/MC/AArch64/label-arithmetic-diags-elf.s | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/test/Sema/asm.c b/clang/test/Sema/asm.c
index 6cd95c71604d44..8976388da3cb84 100644
--- a/clang/test/Sema/asm.c
+++ b/clang/test/Sema/asm.c
@@ -15,7 +15,7 @@ void f(void) {
   asm ("foo\n" : "=a" (i) : "[symbolic_name]" (i)); // expected-error {{invalid input constraint '[symbolic_name]' in asm}}
 
   asm ("foo\n" : : "" (i)); // expected-error {{invalid input constraint '' in asm}}
-  asm ("foo\n" : "=a" (i) : "" (i)); // expected-error {{invalid input constraint '' in asm}}
+  asm ("foo\n" : "=a" (i) : "" (i)); // expected-error {{this will fail}}
 }
 
 void clobbers(void) {
diff --git a/llvm/test/MC/AArch64/label-arithmetic-diags-elf.s b/llvm/test/MC/AArch64/label-arithmetic-diags-elf.s
index 2ef67fafb2ea59..abb1d6a94002fe 100644
--- a/llvm/test/MC/AArch64/label-arithmetic-diags-elf.s
+++ b/llvm/test/MC/AArch64/label-arithmetic-diags-elf.s
@@ -85,3 +85,4 @@ end_across_sec:
   // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: Cannot represent a difference across sections
   // CHECK-NEXT: cmp w0, #(sec_y - sec_x)
   // CHECK-NEXT: ^
+  // CHECK-NEXT: this will fail

>From c72deffbbc7ea7d991966a2e686dd6342ea384c7 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Tue, 22 Oct 2024 14:29:50 +0000
Subject: [PATCH 8/8] Run on our linux agents

---
 .ci/generate-buildkite-pipeline-premerge | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 1a7fe7abbd7a79..f2e94e785e0666 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -293,6 +293,7 @@ if [[ "${linux_projects}" != "" ]]; then
         context: "linux-test-results"
   depends_on: "test Linux"
   allow_dependency_failure: true
+  agents: ${LINUX_AGENTS}
 EOF
 fi
 
@@ -325,6 +326,7 @@ if [[ "${windows_projects}" != "" ]]; then
         context: "windows-test-results"
   depends_on: "test Windows"
   allow_dependency_failure: true
+  agents: ${LINUX_AGENTS}
 EOF
 fi
 



More information about the cfe-commits mailing list