[llvm] [Github][RFC] Add workflow to diff codegen on llvm-test-suite (PR #190010)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 03:39:34 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/190010
>From f208b306fd80f2b0b3240741b44ab639d923b785 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 00:13:05 +0800
Subject: [PATCH 01/30] [Github][RFC] Add workflow to diff codegen on
llvm-test-suite
A common task when reviewing PRs in the LLVM subproject is checking out the PR locally, building it, running it on some benchmarks e.g. llvm-test-suite, and comparing the codegen against some known version.
This is quite laborious though so this PR adds a GitHub workflow to automate the process. It's triggered by commenting "/test-suite" on a PR. The workflow will kick off, build clang with the head and base of the PR, build the benchmarks in llvm-test-suite for several configurations with each version of clang, compute the diff in the output assembly via the tdiff.py script, and then report back with the diffs in a comment.
Here's an example where you can see the diff of a codegen change in the RISC-V backend on my fork: https://github.com/lukel97/llvm-project/pull/5#issuecomment-4171850051
At the moment it's very simple but could be fleshed out later. Currently it builds llvm-test-suite for handful of common configurations, some cross-compiled:
- `-target aarch64-linux-gnu -march=armv9-a -O3`
- `-target riscv64-linux-gnu -march=rva23u64 -O3`
- `-target x86_64-linux-gnu -O3`
We could eventually extend this to accept arbitrary targets and flags in the comment.
It also just comments a link to download the codegen diffs, but could eventually also include some output from the ./utils/compare.py script about e.g. changes in code size or statistics. For now, those results are just uploaded as an artifact.
In terms of worker resources, running it on the free GitHub hosted workers is good enough. Building Clang takes a while, over an hour, but building the test-suite only takes around 10 minutes. But we could stick it on something beefier if we wanted the feedback to be faster.
This workflow requires the PR to be mergeable, as it wants to get the diff of the "mergeability" commit that GitHub generates for each PR. That way the diff is always between the latest version of the base branch and the PR, not the base branch at the time the PR was created.
---
.github/workflows/test-suite.ll | 148 ++++++++++++++++++++++++++++++++
1 file changed, 148 insertions(+)
create mode 100644 .github/workflows/test-suite.ll
diff --git a/.github/workflows/test-suite.ll b/.github/workflows/test-suite.ll
new file mode 100644
index 0000000000000..0d9c81fc74083
--- /dev/null
+++ b/.github/workflows/test-suite.ll
@@ -0,0 +1,148 @@
+# When /test-suite is commented on a PR, checks out the PR, builds clang and
+# then the test-suite in several configurations. It then checks out the base of
+# the PR, builds clang and the test-suite again, and then uploads the diff of
+# the codegen.
+
+name: Diff test-suite codegen
+
+on:
+ issue_comment:
+ types:
+ - created
+
+jobs:
+ test-suite:
+ name: Build and diff
+ runs-on: ubuntu-24.04
+ permissions:
+ issues: write
+ if: >-
+ !startswith(github.event.comment.body, '<!--IGNORE-->') &&
+ github.event.issue.pull_request && contains(github.event.comment.body, '/test-suite')
+ steps:
+ - id: get-pr
+ uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+ with:
+ script: |
+ const { data: pr } = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: context.payload.issue.number
+ })
+ if (!pr.mergeable)
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body: "Can't diff PR, PR isn't mergeable"
+ })
+ return pr
+ - if: ${{ !fromJSON(steps.get-pr.outputs.result).mergeable }}
+ run: exit 1
+ - uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+ with:
+ script: |
+ github.rest.reactions.createForIssueComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: context.payload.comment.id,
+ content: '+1'
+ })
+ - uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+ with:
+ ref: ${{ fromJSON(steps.get-pr.outputs.result).merge_commit_sha }}
+ repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
+ fetch-depth: 2
+ - run: |
+ echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
+ echo "BASE_SHA=$(git rev-parse HEAD^)" >> $GITHUB_ENV
+ - uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+ with:
+ repository: llvm/llvm-test-suite
+ path: llvm-test-suite
+ - name: Install system dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y cmake ninja-build libc6-dev-{arm64,riscv64}-cross libgcc-14-dev-{arm64,riscv64}-cross libstdc++-14-dev-{arm64,riscv64}-cross
+ - name: Configure Clang
+ run: cmake -B build -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD='AArch64;X86;RISCV' -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_APPEND_VC_REV=OFF llvm -GNinja
+ - name: Build Clang @ head
+ run: ninja -C build
+ - name: Configure and build test-suite @ head
+ run: |
+ cat << EOF > rva23u64.cmake
+ set(CMAKE_SYSTEM_NAME Linux)
+ set(CMAKE_C_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang)
+ set(CMAKE_CXX_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang++)
+ set(CMAKE_C_COMPILER_TARGET riscv64-linux-gnu)
+ set(CMAKE_CXX_COMPILER_TARGET riscv64-linux-gnu)
+ set(CMAKE_C_FLAGS_INIT "-march=rva23u64 -save-temps=obj")
+ set(CMAKE_CXX_FLAGS_INIT "-march=rva23u64 -save-temps=obj")
+ set(CMAKE_SYSTEM_PROCESSOR riscv64)
+ set(CMAKE_LINKER_TYPE LLD)
+ EOF
+ cat << EOF > armv9-a.cmake
+ set(CMAKE_SYSTEM_NAME Linux)
+ set(CMAKE_C_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang)
+ set(CMAKE_CXX_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang++)
+ set(CMAKE_C_COMPILER_TARGET aarch64-linux-gnu)
+ set(CMAKE_CXX_COMPILER_TARGET aarch64-linux-gnu)
+ set(CMAKE_C_FLAGS_INIT "-march=armv9-a -save-temps=obj")
+ set(CMAKE_CXX_FLAGS_INIT "-march=armv9-a -save-temps=obj")
+ set(CMAKE_SYSTEM_PROCESSOR arm64)
+ set(CMAKE_LINKER_TYPE LLD)
+ EOF
+ cat << EOF > x86_64.cmake
+ set(CMAKE_C_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang)
+ set(CMAKE_CXX_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang++)
+ set(CMAKE_C_FLAGS_INIT "-save-temps=obj")
+ set(CMAKE_CXX_FLAGS_INIT "-save-temps=obj")
+ set(CMAKE_LINKER_TYPE LLD)
+ EOF
+ build_llvm_test_suite () {
+ cmake -B build.$1 -C cmake/caches/O3.cmake --toolchain $2 -DTEST_SUITE_BENCHMARKING_ONLY=ON -DTEST_SUITE_RUN_BENCHMARKS=OFF -GNinja
+ ninja -C build.$1
+ $GITHUB_WORKSPACE/build/bin/llvm-lit build.$1 -o results.$1.json
+ }
+ build_llvm_test_suite rva23u64-O3-b rva23u64.cmake
+ build_llvm_test_suite armv9-a-O3-b armv9-a.cmake
+ build_llvm_test_suite x86_64-O3-b x86_64.cmake
+ working-directory: llvm-test-suite
+ - name: Build test-suite @ base
+ run: git checkout $BASE_SHA && ninja -C build
+ - name: Configure and build test-suite @ base
+ run: |
+ build_llvm_test_suite () {
+ cmake -B build.$1 -C cmake/caches/O3.cmake --toolchain $2 -DTEST_SUITE_BENCHMARKING_ONLY=ON -DTEST_SUITE_RUN_BENCHMARKS=OFF -GNinja
+ ninja -C build.$1
+ $GITHUB_WORKSPACE/build/bin/llvm-lit build.$1 -o results.$1.json
+ }
+ build_llvm_test_suite rva23u64-O3-a rva23u64.cmake
+ build_llvm_test_suite armv9-a-O3-a armv9-a.cmake
+ build_llvm_test_suite x86_64-O3-a x86_64.cmake
+ working-directory: llvm-test-suite
+ - run: |
+ mkdir diffs
+ ./utils/tdiff.py -a build.rva23u64-O3-a -b build.rva23u64-O3-b -s all > diffs/rva23u64-O3.diff || true
+ ./utils/tdiff.py -a build.armv9-a-O3-a -b build.armv9-a-O3-b -s all > diffs/armv9-a-O3.diff || true
+ ./utils/tdiff.py -a build.x86_64-O3-a -b build.x86_64-O3-b -s all > diffs/x86_64-O3.diff || true
+ working-directory: llvm-test-suite
+ - uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7.0.0
+ id: upload-diffs
+ with:
+ name: diffs
+ path: llvm-test-suite/diffs
+ - uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7.0.0
+ with:
+ name: results
+ path: llvm-test-suite/results*.json
+ - uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+ env:
+ DIFF_URL: ${{ steps.upload-diffs.outputs.artifact-url }}
+ with:
+ script: |
+ github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body: `test-suite diff from ${process.env.BASE_SHA}...${process.env.HEAD_SHA}: ${process.env.DIFF_URL}`
+ })
>From ba5f906c3b42df714931ba8515a519783dd3be86 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 10:55:14 +0800
Subject: [PATCH 02/30] Address review comments
* .ll -> .yml (doh)
* Put cmake files in separate location
* Add bash script for configuring and building llvm-test-suite
---
.../{test-suite.ll => test-suite.yml} | 52 +++----------------
.github/workflows/test-suite/aarch64.cmake | 9 ++++
.../test-suite/configure-and-build.sh | 12 +++++
.github/workflows/test-suite/riscv64.cmake | 9 ++++
.github/workflows/test-suite/x86_64.cmake | 5 ++
5 files changed, 42 insertions(+), 45 deletions(-)
rename .github/workflows/{test-suite.ll => test-suite.yml} (64%)
create mode 100644 .github/workflows/test-suite/aarch64.cmake
create mode 100755 .github/workflows/test-suite/configure-and-build.sh
create mode 100644 .github/workflows/test-suite/riscv64.cmake
create mode 100644 .github/workflows/test-suite/x86_64.cmake
diff --git a/.github/workflows/test-suite.ll b/.github/workflows/test-suite.yml
similarity index 64%
rename from .github/workflows/test-suite.ll
rename to .github/workflows/test-suite.yml
index 0d9c81fc74083..3cd3946a80161 100644
--- a/.github/workflows/test-suite.ll
+++ b/.github/workflows/test-suite.yml
@@ -55,6 +55,7 @@ jobs:
- run: |
echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
echo "BASE_SHA=$(git rev-parse HEAD^)" >> $GITHUB_ENV
+ echo "$GITHUB_WORKSPACE/.github/workflows/test-suite" >> $GITHUB_PATH
- uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
repository: llvm/llvm-test-suite
@@ -69,56 +70,17 @@ jobs:
run: ninja -C build
- name: Configure and build test-suite @ head
run: |
- cat << EOF > rva23u64.cmake
- set(CMAKE_SYSTEM_NAME Linux)
- set(CMAKE_C_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang)
- set(CMAKE_CXX_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang++)
- set(CMAKE_C_COMPILER_TARGET riscv64-linux-gnu)
- set(CMAKE_CXX_COMPILER_TARGET riscv64-linux-gnu)
- set(CMAKE_C_FLAGS_INIT "-march=rva23u64 -save-temps=obj")
- set(CMAKE_CXX_FLAGS_INIT "-march=rva23u64 -save-temps=obj")
- set(CMAKE_SYSTEM_PROCESSOR riscv64)
- set(CMAKE_LINKER_TYPE LLD)
- EOF
- cat << EOF > armv9-a.cmake
- set(CMAKE_SYSTEM_NAME Linux)
- set(CMAKE_C_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang)
- set(CMAKE_CXX_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang++)
- set(CMAKE_C_COMPILER_TARGET aarch64-linux-gnu)
- set(CMAKE_CXX_COMPILER_TARGET aarch64-linux-gnu)
- set(CMAKE_C_FLAGS_INIT "-march=armv9-a -save-temps=obj")
- set(CMAKE_CXX_FLAGS_INIT "-march=armv9-a -save-temps=obj")
- set(CMAKE_SYSTEM_PROCESSOR arm64)
- set(CMAKE_LINKER_TYPE LLD)
- EOF
- cat << EOF > x86_64.cmake
- set(CMAKE_C_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang)
- set(CMAKE_CXX_COMPILER ${GITHUB_WORKSPACE}/build/bin/clang++)
- set(CMAKE_C_FLAGS_INIT "-save-temps=obj")
- set(CMAKE_CXX_FLAGS_INIT "-save-temps=obj")
- set(CMAKE_LINKER_TYPE LLD)
- EOF
- build_llvm_test_suite () {
- cmake -B build.$1 -C cmake/caches/O3.cmake --toolchain $2 -DTEST_SUITE_BENCHMARKING_ONLY=ON -DTEST_SUITE_RUN_BENCHMARKS=OFF -GNinja
- ninja -C build.$1
- $GITHUB_WORKSPACE/build/bin/llvm-lit build.$1 -o results.$1.json
- }
- build_llvm_test_suite rva23u64-O3-b rva23u64.cmake
- build_llvm_test_suite armv9-a-O3-b armv9-a.cmake
- build_llvm_test_suite x86_64-O3-b x86_64.cmake
+ build-and-configure.sh rva23u64-O3-b riscv64.cmake
+ build-and-configure.sh armv9-a-O3-b aarch64.cmake
+ build-and-configure.sh x86_64-O3-b x86_64.cmake
working-directory: llvm-test-suite
- name: Build test-suite @ base
run: git checkout $BASE_SHA && ninja -C build
- name: Configure and build test-suite @ base
run: |
- build_llvm_test_suite () {
- cmake -B build.$1 -C cmake/caches/O3.cmake --toolchain $2 -DTEST_SUITE_BENCHMARKING_ONLY=ON -DTEST_SUITE_RUN_BENCHMARKS=OFF -GNinja
- ninja -C build.$1
- $GITHUB_WORKSPACE/build/bin/llvm-lit build.$1 -o results.$1.json
- }
- build_llvm_test_suite rva23u64-O3-a rva23u64.cmake
- build_llvm_test_suite armv9-a-O3-a armv9-a.cmake
- build_llvm_test_suite x86_64-O3-a x86_64.cmake
+ build-and-configure.sh rva23u64-O3-a riscv64.cmake
+ build-and-configure.sh armv9-a-O3-a aarch64.cmake
+ build-and-configure.sh x86_64-O3-a x86_64.cmake
working-directory: llvm-test-suite
- run: |
mkdir diffs
diff --git a/.github/workflows/test-suite/aarch64.cmake b/.github/workflows/test-suite/aarch64.cmake
new file mode 100644
index 0000000000000..4215cf8e9d5eb
--- /dev/null
+++ b/.github/workflows/test-suite/aarch64.cmake
@@ -0,0 +1,9 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang)
+set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang++)
+set(CMAKE_C_COMPILER_TARGET aarch64-linux-gnu)
+set(CMAKE_CXX_COMPILER_TARGET aarch64-linux-gnu)
+set(CMAKE_C_FLAGS_INIT "-march=armv9-a -save-temps=obj")
+set(CMAKE_CXX_FLAGS_INIT "-march=armv9-a -save-temps=obj")
+set(CMAKE_SYSTEM_PROCESSOR arm64)
+set(CMAKE_LINKER_TYPE LLD)
diff --git a/.github/workflows/test-suite/configure-and-build.sh b/.github/workflows/test-suite/configure-and-build.sh
new file mode 100755
index 0000000000000..248e408e1ca9b
--- /dev/null
+++ b/.github/workflows/test-suite/configure-and-build.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eux
+
+cmake -B build.$1 \
+ --toolchain $GITHUB_WORKSPACE/.github/workflows/test-suite/$2 \
+ -C cmake/caches/O3.cmake \
+ -GNinja \
+ -DTEST_SUITE_BENCHMARKING_ONLY=ON \
+ -DTEST_SUITE_RUN_BENCHMARKS=OFF
+ninja -C build.$1
+$GITHUB_WORKSPACE/build/bin/llvm-lit build.$1 -o results.$1.json
diff --git a/.github/workflows/test-suite/riscv64.cmake b/.github/workflows/test-suite/riscv64.cmake
new file mode 100644
index 0000000000000..19f924bcd6003
--- /dev/null
+++ b/.github/workflows/test-suite/riscv64.cmake
@@ -0,0 +1,9 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang)
+set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang++)
+set(CMAKE_C_COMPILER_TARGET riscv64-linux-gnu)
+set(CMAKE_CXX_COMPILER_TARGET riscv64-linux-gnu)
+set(CMAKE_C_FLAGS_INIT "-march=rva23u64 -save-temps=obj")
+set(CMAKE_CXX_FLAGS_INIT "-march=rva23u64 -save-temps=obj")
+set(CMAKE_SYSTEM_PROCESSOR riscv64)
+set(CMAKE_LINKER_TYPE LLD)
diff --git a/.github/workflows/test-suite/x86_64.cmake b/.github/workflows/test-suite/x86_64.cmake
new file mode 100644
index 0000000000000..d1150ac1a42e7
--- /dev/null
+++ b/.github/workflows/test-suite/x86_64.cmake
@@ -0,0 +1,5 @@
+set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang)
+set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang++)
+set(CMAKE_C_FLAGS_INIT "-save-temps=obj")
+set(CMAKE_CXX_FLAGS_INIT "-save-temps=obj")
+set(CMAKE_LINKER_TYPE LLD)
>From 68707da10b18f55a403724133986c0911fe7f9b0 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 11:04:55 +0800
Subject: [PATCH 03/30] Fix permissions
---
.github/workflows/test-suite.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 3cd3946a80161..2a0725d43625c 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -15,7 +15,7 @@ jobs:
name: Build and diff
runs-on: ubuntu-24.04
permissions:
- issues: write
+ pull-requests: write
if: >-
!startswith(github.event.comment.body, '<!--IGNORE-->') &&
github.event.issue.pull_request && contains(github.event.comment.body, '/test-suite')
>From af377f061e65adf6b1777de5c4de5a1659feb4a4 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 11:30:14 +0800
Subject: [PATCH 04/30] Use scripts + cmake files from default branch of
upstream repo
Instead of using untrusted PR repo
---
.github/workflows/test-suite.yml | 58 +++++++++++++------
.../test-suite/configure-and-build.sh | 2 +-
2 files changed, 40 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 2a0725d43625c..d6048bf05a93f 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -20,7 +20,8 @@ jobs:
!startswith(github.event.comment.body, '<!--IGNORE-->') &&
github.event.issue.pull_request && contains(github.event.comment.body, '/test-suite')
steps:
- - id: get-pr
+ - name: Get pull request
+ id: get-pr
uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
script: |
@@ -36,9 +37,11 @@ jobs:
body: "Can't diff PR, PR isn't mergeable"
})
return pr
- - if: ${{ !fromJSON(steps.get-pr.outputs.result).mergeable }}
+ - name: Check pull request is mergeable
+ if: ${{ !fromJSON(steps.get-pr.outputs.result).mergeable }}
run: exit 1
- - uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+ - name: Thumbs up comment
+ uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
script: |
github.rest.reactions.createForIssueComment({
@@ -47,19 +50,32 @@ jobs:
comment_id: context.payload.comment.id,
content: '+1'
})
- - uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+ - name: Checkout pull request
+ uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
ref: ${{ fromJSON(steps.get-pr.outputs.result).merge_commit_sha }}
repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
fetch-depth: 2
- - run: |
- echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
- echo "BASE_SHA=$(git rev-parse HEAD^)" >> $GITHUB_ENV
- echo "$GITHUB_WORKSPACE/.github/workflows/test-suite" >> $GITHUB_PATH
- - uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+ # Check out the test-suite scripts and CMake files from the default branch
+ # on the upstream repository. Use these instead of the scripts in the pull
+ # request's branch since it's untrusted.
+ - name: Checkout scripts and CMake files
+ uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+ with:
+ path: test-suite
+ sparse-checkout: .github/workflows/test-suite
+ - name: Checkout llvm/llvm-test-suite
+ uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
repository: llvm/llvm-test-suite
path: llvm-test-suite
+ - name: Setup environment variables
+ run: |
+ echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
+ echo "BASE_SHA=$(git rev-parse HEAD^)" >> $GITHUB_ENV
+ SCRIPTS_DIR="$GITHUB_WORKSPACE/test-suite/.github/workflows/test-suite"
+ echo "SCRIPTS_DIR=$SCRIPTS_DIR" >> $GITHUB_ENV
+ echo "$SCRIPTS_DIR" >> $GITHUB_PATH
- name: Install system dependencies
run: |
sudo apt-get update
@@ -70,34 +86,38 @@ jobs:
run: ninja -C build
- name: Configure and build test-suite @ head
run: |
- build-and-configure.sh rva23u64-O3-b riscv64.cmake
- build-and-configure.sh armv9-a-O3-b aarch64.cmake
- build-and-configure.sh x86_64-O3-b x86_64.cmake
+ build-and-configure.sh rva23u64-O3-b $SCRIPTS_DIR/riscv64.cmake
+ build-and-configure.sh armv9-a-O3-b $SCRIPTS_DIR/aarch64.cmake
+ build-and-configure.sh x86_64-O3-b $SCRIPTS_DIR/x86_64.cmake
working-directory: llvm-test-suite
- name: Build test-suite @ base
run: git checkout $BASE_SHA && ninja -C build
- name: Configure and build test-suite @ base
run: |
- build-and-configure.sh rva23u64-O3-a riscv64.cmake
- build-and-configure.sh armv9-a-O3-a aarch64.cmake
- build-and-configure.sh x86_64-O3-a x86_64.cmake
+ build-and-configure.sh rva23u64-O3-a $SCRIPTS_DIR/riscv64.cmake
+ build-and-configure.sh armv9-a-O3-a $SCRIPTS_DIR/aarch64.cmake
+ build-and-configure.sh x86_64-O3-a $SCRIPTS_DIR/x86_64.cmake
working-directory: llvm-test-suite
- - run: |
+ - name: Compute diffs
+ run: |
mkdir diffs
./utils/tdiff.py -a build.rva23u64-O3-a -b build.rva23u64-O3-b -s all > diffs/rva23u64-O3.diff || true
./utils/tdiff.py -a build.armv9-a-O3-a -b build.armv9-a-O3-b -s all > diffs/armv9-a-O3.diff || true
./utils/tdiff.py -a build.x86_64-O3-a -b build.x86_64-O3-b -s all > diffs/x86_64-O3.diff || true
working-directory: llvm-test-suite
- - uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7.0.0
+ - name: Upload diffs
+ uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7.0.0
id: upload-diffs
with:
name: diffs
path: llvm-test-suite/diffs
- - uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7.0.0
+ - name: Upload results
+ uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7.0.0
with:
name: results
path: llvm-test-suite/results*.json
- - uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+ - name: Create comment
+ uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
env:
DIFF_URL: ${{ steps.upload-diffs.outputs.artifact-url }}
with:
diff --git a/.github/workflows/test-suite/configure-and-build.sh b/.github/workflows/test-suite/configure-and-build.sh
index 248e408e1ca9b..14303a5015493 100755
--- a/.github/workflows/test-suite/configure-and-build.sh
+++ b/.github/workflows/test-suite/configure-and-build.sh
@@ -3,7 +3,7 @@
set -eux
cmake -B build.$1 \
- --toolchain $GITHUB_WORKSPACE/.github/workflows/test-suite/$2 \
+ --toolchain $2 \
-C cmake/caches/O3.cmake \
-GNinja \
-DTEST_SUITE_BENCHMARKING_ONLY=ON \
>From 56563cf712eed0ff1a4a7c4fab0c17a82e01cf4c Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 13:10:50 +0800
Subject: [PATCH 05/30] Fix typo
---
.github/workflows/test-suite.yml | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index d6048bf05a93f..59b0e9196ddbf 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -86,17 +86,17 @@ jobs:
run: ninja -C build
- name: Configure and build test-suite @ head
run: |
- build-and-configure.sh rva23u64-O3-b $SCRIPTS_DIR/riscv64.cmake
- build-and-configure.sh armv9-a-O3-b $SCRIPTS_DIR/aarch64.cmake
- build-and-configure.sh x86_64-O3-b $SCRIPTS_DIR/x86_64.cmake
+ configure-and-build.sh rva23u64-O3-b $SCRIPTS_DIR/riscv64.cmake
+ configure-and-build.sh armv9-a-O3-b $SCRIPTS_DIR/aarch64.cmake
+ configure-and-build.sh x86_64-O3-b $SCRIPTS_DIR/x86_64.cmake
working-directory: llvm-test-suite
- name: Build test-suite @ base
run: git checkout $BASE_SHA && ninja -C build
- name: Configure and build test-suite @ base
run: |
- build-and-configure.sh rva23u64-O3-a $SCRIPTS_DIR/riscv64.cmake
- build-and-configure.sh armv9-a-O3-a $SCRIPTS_DIR/aarch64.cmake
- build-and-configure.sh x86_64-O3-a $SCRIPTS_DIR/x86_64.cmake
+ configure-and-build.sh rva23u64-O3-a $SCRIPTS_DIR/riscv64.cmake
+ configure-and-build.sh armv9-a-O3-a $SCRIPTS_DIR/aarch64.cmake
+ configure-and-build.sh x86_64-O3-a $SCRIPTS_DIR/x86_64.cmake
working-directory: llvm-test-suite
- name: Compute diffs
run: |
>From 4465c795ccb375a2d4d5232405e9105d64e28c9f Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 15:21:40 +0800
Subject: [PATCH 06/30] Rename a and b to head and base
---
.github/workflows/test-suite.yml | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 59b0e9196ddbf..07410f4208e14 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -86,24 +86,24 @@ jobs:
run: ninja -C build
- name: Configure and build test-suite @ head
run: |
- configure-and-build.sh rva23u64-O3-b $SCRIPTS_DIR/riscv64.cmake
- configure-and-build.sh armv9-a-O3-b $SCRIPTS_DIR/aarch64.cmake
- configure-and-build.sh x86_64-O3-b $SCRIPTS_DIR/x86_64.cmake
+ configure-and-build.sh rva23u64-O3-head $SCRIPTS_DIR/riscv64.cmake
+ configure-and-build.sh armv9-a-O3-head $SCRIPTS_DIR/aarch64.cmake
+ configure-and-build.sh x86_64-O3-head $SCRIPTS_DIR/x86_64.cmake
working-directory: llvm-test-suite
- name: Build test-suite @ base
run: git checkout $BASE_SHA && ninja -C build
- name: Configure and build test-suite @ base
run: |
- configure-and-build.sh rva23u64-O3-a $SCRIPTS_DIR/riscv64.cmake
- configure-and-build.sh armv9-a-O3-a $SCRIPTS_DIR/aarch64.cmake
- configure-and-build.sh x86_64-O3-a $SCRIPTS_DIR/x86_64.cmake
+ configure-and-build.sh rva23u64-O3-base $SCRIPTS_DIR/riscv64.cmake
+ configure-and-build.sh armv9-a-O3-base $SCRIPTS_DIR/aarch64.cmake
+ configure-and-build.sh x86_64-O3-base $SCRIPTS_DIR/x86_64.cmake
working-directory: llvm-test-suite
- name: Compute diffs
run: |
mkdir diffs
- ./utils/tdiff.py -a build.rva23u64-O3-a -b build.rva23u64-O3-b -s all > diffs/rva23u64-O3.diff || true
- ./utils/tdiff.py -a build.armv9-a-O3-a -b build.armv9-a-O3-b -s all > diffs/armv9-a-O3.diff || true
- ./utils/tdiff.py -a build.x86_64-O3-a -b build.x86_64-O3-b -s all > diffs/x86_64-O3.diff || true
+ ./utils/tdiff.py -a build.rva23u64-O3-base -b build.rva23u64-O3-head -s all > diffs/rva23u64-O3.diff || true
+ ./utils/tdiff.py -a build.armv9-a-O3-base -b build.armv9-a-O3-head -s all > diffs/armv9-a-O3.diff || true
+ ./utils/tdiff.py -a build.x86_64-O3-base -b build.x86_64-O3-head -s all > diffs/x86_64-O3.diff || true
working-directory: llvm-test-suite
- name: Upload diffs
uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7.0.0
>From fc12789fbb053fcf4d902a89610f24034d630578 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 15:32:49 +0800
Subject: [PATCH 07/30] Use cmake cache for configuring llvm
---
.github/workflows/test-suite.yml | 6 +++---
.github/workflows/test-suite/llvm.cmake | 4 ++++
2 files changed, 7 insertions(+), 3 deletions(-)
create mode 100644 .github/workflows/test-suite/llvm.cmake
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 07410f4208e14..0a0e2a6563e76 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -62,7 +62,7 @@ jobs:
- name: Checkout scripts and CMake files
uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
- path: test-suite
+ path: scripts
sparse-checkout: .github/workflows/test-suite
- name: Checkout llvm/llvm-test-suite
uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
@@ -73,7 +73,7 @@ jobs:
run: |
echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
echo "BASE_SHA=$(git rev-parse HEAD^)" >> $GITHUB_ENV
- SCRIPTS_DIR="$GITHUB_WORKSPACE/test-suite/.github/workflows/test-suite"
+ SCRIPTS_DIR="$GITHUB_WORKSPACE/scripts/.github/workflows/test-suite"
echo "SCRIPTS_DIR=$SCRIPTS_DIR" >> $GITHUB_ENV
echo "$SCRIPTS_DIR" >> $GITHUB_PATH
- name: Install system dependencies
@@ -81,7 +81,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y cmake ninja-build libc6-dev-{arm64,riscv64}-cross libgcc-14-dev-{arm64,riscv64}-cross libstdc++-14-dev-{arm64,riscv64}-cross
- name: Configure Clang
- run: cmake -B build -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD='AArch64;X86;RISCV' -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_APPEND_VC_REV=OFF llvm -GNinja
+ run: cmake -B build -C $SCRIPTS_DIR/llvm.cmake llvm -GNinja
- name: Build Clang @ head
run: ninja -C build
- name: Configure and build test-suite @ head
diff --git a/.github/workflows/test-suite/llvm.cmake b/.github/workflows/test-suite/llvm.cmake
new file mode 100644
index 0000000000000..20d21e7f3495d
--- /dev/null
+++ b/.github/workflows/test-suite/llvm.cmake
@@ -0,0 +1,4 @@
+set(CMAKE_BUILD_TYPE Release CACHE STRING "")
+set(LLVM_TARGETS_TO_BUILD "AArch64;RISCV;X86" CACHE STRING "")
+set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
+set(LLVM_APPEND_VC_REF OFF CACHE STRING "")
>From 0cd6c26eee8469b41b0f4495448cffde9d0c3ec0 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 15:37:56 +0800
Subject: [PATCH 08/30] Clone llvm-project in a separate directory
So we don't need to worry about any of the other clones clobbering anything
---
.github/workflows/test-suite.yml | 5 +++++
.github/workflows/test-suite/aarch64.cmake | 4 ++--
.github/workflows/test-suite/riscv64.cmake | 4 ++--
.github/workflows/test-suite/x86_64.cmake | 4 ++--
4 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 0a0e2a6563e76..92bcb858ee9ec 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -56,6 +56,7 @@ jobs:
ref: ${{ fromJSON(steps.get-pr.outputs.result).merge_commit_sha }}
repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
fetch-depth: 2
+ path: llvm-project
# Check out the test-suite scripts and CMake files from the default branch
# on the upstream repository. Use these instead of the scripts in the pull
# request's branch since it's untrusted.
@@ -76,14 +77,17 @@ jobs:
SCRIPTS_DIR="$GITHUB_WORKSPACE/scripts/.github/workflows/test-suite"
echo "SCRIPTS_DIR=$SCRIPTS_DIR" >> $GITHUB_ENV
echo "$SCRIPTS_DIR" >> $GITHUB_PATH
+ working-directory: llvm-project
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y cmake ninja-build libc6-dev-{arm64,riscv64}-cross libgcc-14-dev-{arm64,riscv64}-cross libstdc++-14-dev-{arm64,riscv64}-cross
- name: Configure Clang
run: cmake -B build -C $SCRIPTS_DIR/llvm.cmake llvm -GNinja
+ working-directory: llvm-project
- name: Build Clang @ head
run: ninja -C build
+ working-directory: llvm-project
- name: Configure and build test-suite @ head
run: |
configure-and-build.sh rva23u64-O3-head $SCRIPTS_DIR/riscv64.cmake
@@ -92,6 +96,7 @@ jobs:
working-directory: llvm-test-suite
- name: Build test-suite @ base
run: git checkout $BASE_SHA && ninja -C build
+ working-directory: llvm-project
- name: Configure and build test-suite @ base
run: |
configure-and-build.sh rva23u64-O3-base $SCRIPTS_DIR/riscv64.cmake
diff --git a/.github/workflows/test-suite/aarch64.cmake b/.github/workflows/test-suite/aarch64.cmake
index 4215cf8e9d5eb..ecc69bb5e0f3c 100644
--- a/.github/workflows/test-suite/aarch64.cmake
+++ b/.github/workflows/test-suite/aarch64.cmake
@@ -1,6 +1,6 @@
set(CMAKE_SYSTEM_NAME Linux)
-set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang)
-set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang++)
+set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/llvm-project/build/bin/clang)
+set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/llvm-project/build/bin/clang++)
set(CMAKE_C_COMPILER_TARGET aarch64-linux-gnu)
set(CMAKE_CXX_COMPILER_TARGET aarch64-linux-gnu)
set(CMAKE_C_FLAGS_INIT "-march=armv9-a -save-temps=obj")
diff --git a/.github/workflows/test-suite/riscv64.cmake b/.github/workflows/test-suite/riscv64.cmake
index 19f924bcd6003..47499a6e14ec7 100644
--- a/.github/workflows/test-suite/riscv64.cmake
+++ b/.github/workflows/test-suite/riscv64.cmake
@@ -1,6 +1,6 @@
set(CMAKE_SYSTEM_NAME Linux)
-set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang)
-set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang++)
+set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/llvm-project/build/bin/clang)
+set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/llvm-project/build/bin/clang++)
set(CMAKE_C_COMPILER_TARGET riscv64-linux-gnu)
set(CMAKE_CXX_COMPILER_TARGET riscv64-linux-gnu)
set(CMAKE_C_FLAGS_INIT "-march=rva23u64 -save-temps=obj")
diff --git a/.github/workflows/test-suite/x86_64.cmake b/.github/workflows/test-suite/x86_64.cmake
index d1150ac1a42e7..943630267dfcc 100644
--- a/.github/workflows/test-suite/x86_64.cmake
+++ b/.github/workflows/test-suite/x86_64.cmake
@@ -1,5 +1,5 @@
-set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang)
-set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/build/bin/clang++)
+set(CMAKE_C_COMPILER $ENV{GITHUB_WORKSPACE}/llvm-project/build/bin/clang)
+set(CMAKE_CXX_COMPILER $ENV{GITHUB_WORKSPACE}/llvm-project/build/bin/clang++)
set(CMAKE_C_FLAGS_INIT "-save-temps=obj")
set(CMAKE_CXX_FLAGS_INIT "-save-temps=obj")
set(CMAKE_LINKER_TYPE LLD)
>From 387d32b4c5315ffb9d4234124f515da9c9d97550 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 18:28:49 +0800
Subject: [PATCH 09/30] Add lit to path
---
.github/workflows/test-suite.yml | 1 +
.github/workflows/test-suite/configure-and-build.sh | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 92bcb858ee9ec..70d935b73584b 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -77,6 +77,7 @@ jobs:
SCRIPTS_DIR="$GITHUB_WORKSPACE/scripts/.github/workflows/test-suite"
echo "SCRIPTS_DIR=$SCRIPTS_DIR" >> $GITHUB_ENV
echo "$SCRIPTS_DIR" >> $GITHUB_PATH
+ echo "$GITHUB_WORKSPACE/llvm-project/build/bin" >> $GITHUB_PATH
working-directory: llvm-project
- name: Install system dependencies
run: |
diff --git a/.github/workflows/test-suite/configure-and-build.sh b/.github/workflows/test-suite/configure-and-build.sh
index 14303a5015493..26d58a248a6fd 100755
--- a/.github/workflows/test-suite/configure-and-build.sh
+++ b/.github/workflows/test-suite/configure-and-build.sh
@@ -9,4 +9,4 @@ cmake -B build.$1 \
-DTEST_SUITE_BENCHMARKING_ONLY=ON \
-DTEST_SUITE_RUN_BENCHMARKS=OFF
ninja -C build.$1
-$GITHUB_WORKSPACE/build/bin/llvm-lit build.$1 -o results.$1.json
+llvm-lit build.$1 -o results.$1.json
>From 9a18f4aa97e33ae15e8bfddd280944d981f585d0 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 3 Apr 2026 00:08:26 +0800
Subject: [PATCH 10/30] Split issue commenting into other workflow, drop
pull_request: write permission
---
.github/workflows/issue-write.yml | 1 +
.github/workflows/test-suite.yml | 52 ++++++++-----------------------
2 files changed, 14 insertions(+), 39 deletions(-)
diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml
index db378e286932d..36c5877d87309 100644
--- a/.github/workflows/issue-write.yml
+++ b/.github/workflows/issue-write.yml
@@ -10,6 +10,7 @@ on:
- "CI Checks"
- "Test Issue Write"
- "Check LLVM ABI annotations"
+ - "Diff test-suite codegen"
types:
- completed
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 70d935b73584b..edba6ab793eea 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -12,10 +12,8 @@ on:
jobs:
test-suite:
- name: Build and diff
+ name: Build test-suite and diff
runs-on: ubuntu-24.04
- permissions:
- pull-requests: write
if: >-
!startswith(github.event.comment.body, '<!--IGNORE-->') &&
github.event.issue.pull_request && contains(github.event.comment.body, '/test-suite')
@@ -30,26 +28,12 @@ jobs:
repo: context.repo.repo,
pull_number: context.payload.issue.number
})
- if (!pr.mergeable)
- await github.rest.issues.createComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- body: "Can't diff PR, PR isn't mergeable"
- })
return pr
- name: Check pull request is mergeable
if: ${{ !fromJSON(steps.get-pr.outputs.result).mergeable }}
- run: exit 1
- - name: Thumbs up comment
- uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
- with:
- script: |
- github.rest.reactions.createForIssueComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- comment_id: context.payload.comment.id,
- content: '+1'
- })
+ run: |
+ echo "Unable to diff test-suite with PR, PR isn't mergeable" >> comments
+ exit 1
- name: Checkout pull request
uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
@@ -57,14 +41,6 @@ jobs:
repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
fetch-depth: 2
path: llvm-project
- # Check out the test-suite scripts and CMake files from the default branch
- # on the upstream repository. Use these instead of the scripts in the pull
- # request's branch since it's untrusted.
- - name: Checkout scripts and CMake files
- uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- with:
- path: scripts
- sparse-checkout: .github/workflows/test-suite
- name: Checkout llvm/llvm-test-suite
uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
@@ -74,7 +50,7 @@ jobs:
run: |
echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
echo "BASE_SHA=$(git rev-parse HEAD^)" >> $GITHUB_ENV
- SCRIPTS_DIR="$GITHUB_WORKSPACE/scripts/.github/workflows/test-suite"
+ SCRIPTS_DIR="$GITHUB_WORKSPACE/llvm-project/.github/workflows/test-suite"
echo "SCRIPTS_DIR=$SCRIPTS_DIR" >> $GITHUB_ENV
echo "$SCRIPTS_DIR" >> $GITHUB_PATH
echo "$GITHUB_WORKSPACE/llvm-project/build/bin" >> $GITHUB_PATH
@@ -123,14 +99,12 @@ jobs:
name: results
path: llvm-test-suite/results*.json
- name: Create comment
- uses: actions/github-script at ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
- env:
- DIFF_URL: ${{ steps.upload-diffs.outputs.artifact-url }}
+ run: |
+ echo "test-suite diff from $BASE_SHA...$HEAD_SHA: ${{ steps.upload-diffs.outputs.artifact-url }} >> comments
+ - name: Upload comment
+ uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+ if: always()
with:
- script: |
- github.rest.issues.createComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: context.issue.number,
- body: `test-suite diff from ${process.env.BASE_SHA}...${process.env.HEAD_SHA}: ${process.env.DIFF_URL}`
- })
+ name: workflow-args
+ path: |
+ comments
>From 6fccd7bdfbc12f7a301bd67dfcaf0273afc6e4de Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 3 Apr 2026 00:09:05 +0800
Subject: [PATCH 11/30] persist-credentials: false
---
.github/workflows/test-suite.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index edba6ab793eea..b4fc9ab058e49 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -41,11 +41,13 @@ jobs:
repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
fetch-depth: 2
path: llvm-project
+ persist-credentials: false
- name: Checkout llvm/llvm-test-suite
uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
repository: llvm/llvm-test-suite
path: llvm-test-suite
+ persist-credentials: false
- name: Setup environment variables
run: |
echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
>From 765dc04af15dc8a7e64be2ba023231aef9825d48 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 3 Apr 2026 00:09:24 +0800
Subject: [PATCH 12/30] permissions: { contents: read }
---
.github/workflows/test-suite.yml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index b4fc9ab058e49..b05c12fe3ae93 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -10,6 +10,9 @@ on:
types:
- created
+permissions:
+ contents: read
+
jobs:
test-suite:
name: Build test-suite and diff
>From aa01aa3c59658625fec95734448824439636aac3 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 3 Apr 2026 00:11:55 +0800
Subject: [PATCH 13/30] Only accept comments that start with /test-suite
---
.github/workflows/test-suite.yml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index b05c12fe3ae93..600bd967535b0 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -17,9 +17,7 @@ jobs:
test-suite:
name: Build test-suite and diff
runs-on: ubuntu-24.04
- if: >-
- !startswith(github.event.comment.body, '<!--IGNORE-->') &&
- github.event.issue.pull_request && contains(github.event.comment.body, '/test-suite')
+ if: github.event.issue.pull_request && startswith(github.event.comment.body, '/test-suite')
steps:
- name: Get pull request
id: get-pr
>From ad54facab661ba32cf0250afedc4560dc983f5b7 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 3 Apr 2026 00:12:11 +0800
Subject: [PATCH 14/30] Use CI image
---
.github/workflows/test-suite.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 600bd967535b0..580d7d804c056 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -17,6 +17,8 @@ jobs:
test-suite:
name: Build test-suite and diff
runs-on: ubuntu-24.04
+ container:
+ image: ghcr.io/llvm/ci-ubuntu-24.04:latest
if: github.event.issue.pull_request && startswith(github.event.comment.body, '/test-suite')
steps:
- name: Get pull request
>From fc4e6af3f2eed205d336ce85068258a75f840f1d Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 3 Apr 2026 00:18:54 +0800
Subject: [PATCH 15/30] Allow issue_comment in issue-write so we can use it
from test-suite.yml
---
.github/workflows/issue-write.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml
index 36c5877d87309..4de4e36dd3e03 100644
--- a/.github/workflows/issue-write.yml
+++ b/.github/workflows/issue-write.yml
@@ -23,7 +23,8 @@ jobs:
permissions:
pull-requests: write
if: >
- github.event.workflow_run.event == 'pull_request' &&
+ (github.event.workflow_run.event == 'pull_request' ||
+ github.event.workflow_run.event == 'issue_comment') &&
(
github.event.workflow_run.conclusion == 'success' ||
github.event.workflow_run.conclusion == 'failure'
>From 4aec0909219d1edb349ee7b799e1457660f27ec2 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 3 Apr 2026 00:23:49 +0800
Subject: [PATCH 16/30] Use /bin/sh since CI image doesn't have bash
---
.github/workflows/test-suite.yml | 2 +-
.github/workflows/test-suite/configure-and-build.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 580d7d804c056..02234c7ac44f4 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -63,7 +63,7 @@ jobs:
- name: Install system dependencies
run: |
sudo apt-get update
- sudo apt-get install -y cmake ninja-build libc6-dev-{arm64,riscv64}-cross libgcc-14-dev-{arm64,riscv64}-cross libstdc++-14-dev-{arm64,riscv64}-cross
+ sudo apt-get install -y cmake ninja-build libc6-dev-arm64-cross libc6-dev-riscv64-cross libgcc-14-dev-arm64-cross libgcc-14-dev-riscv64-cross libstdc++-14-dev-arm64-cross libstdc++-14-dev-riscv64-cross
- name: Configure Clang
run: cmake -B build -C $SCRIPTS_DIR/llvm.cmake llvm -GNinja
working-directory: llvm-project
diff --git a/.github/workflows/test-suite/configure-and-build.sh b/.github/workflows/test-suite/configure-and-build.sh
index 26d58a248a6fd..3b78f88f9c6ca 100755
--- a/.github/workflows/test-suite/configure-and-build.sh
+++ b/.github/workflows/test-suite/configure-and-build.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
set -eux
>From 3505249d8ff54d137f325bc054f13382b20c84e3 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 3 Apr 2026 00:36:59 +0800
Subject: [PATCH 17/30] Add missing "
---
.github/workflows/test-suite.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 02234c7ac44f4..c3a239304bd12 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -105,7 +105,7 @@ jobs:
path: llvm-test-suite/results*.json
- name: Create comment
run: |
- echo "test-suite diff from $BASE_SHA...$HEAD_SHA: ${{ steps.upload-diffs.outputs.artifact-url }} >> comments
+ echo "test-suite diff from $BASE_SHA...$HEAD_SHA: ${{ steps.upload-diffs.outputs.artifact-url }}" >> comments
- name: Upload comment
uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
>From cc3e4c677795bb8e5a9bbe41af0b1bba150f4da1 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 6 Apr 2026 10:36:01 +0800
Subject: [PATCH 18/30] Update CMake
---
.github/workflows/test-suite.yml | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index c3a239304bd12..12648af8a730b 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -62,8 +62,14 @@ jobs:
working-directory: llvm-project
- name: Install system dependencies
run: |
+ # Install newer version of CMake (llvm/ci-ubuntu-24.04 image has CMake 3.28)
sudo apt-get update
- sudo apt-get install -y cmake ninja-build libc6-dev-arm64-cross libc6-dev-riscv64-cross libgcc-14-dev-arm64-cross libgcc-14-dev-riscv64-cross libstdc++-14-dev-arm64-cross libstdc++-14-dev-riscv64-cross
+ sudo apt-get install -y wget
+ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+ echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
+
+ sudo apt-get update
+ sudo apt-get install -y cmake libc6-dev-arm64-cross libc6-dev-riscv64-cross libgcc-14-dev-arm64-cross libgcc-14-dev-riscv64-cross libstdc++-14-dev-arm64-cross libstdc++-14-dev-riscv64-cross
- name: Configure Clang
run: cmake -B build -C $SCRIPTS_DIR/llvm.cmake llvm -GNinja
working-directory: llvm-project
>From 79560bbd665ae91c7c8b4058301b88f43f5d2a29 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 6 Apr 2026 11:46:04 +0800
Subject: [PATCH 19/30] Install tcl, needed to build llvm-test-suite
---
.github/workflows/test-suite.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 12648af8a730b..2e7f422d4539a 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -69,7 +69,7 @@ jobs:
echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
sudo apt-get update
- sudo apt-get install -y cmake libc6-dev-arm64-cross libc6-dev-riscv64-cross libgcc-14-dev-arm64-cross libgcc-14-dev-riscv64-cross libstdc++-14-dev-arm64-cross libstdc++-14-dev-riscv64-cross
+ sudo apt-get install -y cmake tcl libc6-dev-arm64-cross libc6-dev-riscv64-cross libgcc-14-dev-arm64-cross libgcc-14-dev-riscv64-cross libstdc++-14-dev-arm64-cross libstdc++-14-dev-riscv64-cross
- name: Configure Clang
run: cmake -B build -C $SCRIPTS_DIR/llvm.cmake llvm -GNinja
working-directory: llvm-project
>From 95636943a469b5092d79759d2c0b5006d9a8da72 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 6 Apr 2026 13:05:17 +0800
Subject: [PATCH 20/30] Fix typo
---
.github/workflows/test-suite.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 2e7f422d4539a..d4d3286d231a5 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -82,7 +82,7 @@ jobs:
configure-and-build.sh armv9-a-O3-head $SCRIPTS_DIR/aarch64.cmake
configure-and-build.sh x86_64-O3-head $SCRIPTS_DIR/x86_64.cmake
working-directory: llvm-test-suite
- - name: Build test-suite @ base
+ - name: Build Clang @ base
run: git checkout $BASE_SHA && ninja -C build
working-directory: llvm-project
- name: Configure and build test-suite @ base
>From e16151cb6ab8004bb1d6ec869303c87cc26697bc Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 7 Apr 2026 15:36:56 +0800
Subject: [PATCH 21/30] Fix comments file not being json
---
.github/workflows/test-suite.yml | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index d4d3286d231a5..0f90c63e29ce7 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -35,7 +35,9 @@ jobs:
- name: Check pull request is mergeable
if: ${{ !fromJSON(steps.get-pr.outputs.result).mergeable }}
run: |
- echo "Unable to diff test-suite with PR, PR isn't mergeable" >> comments
+ cat << EOF > comments
+ [{"body": "Unable to diff test-suite with PR, PR isn't mergeable"}]
+ EOF
exit 1
- name: Checkout pull request
uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
@@ -111,7 +113,9 @@ jobs:
path: llvm-test-suite/results*.json
- name: Create comment
run: |
- echo "test-suite diff from $BASE_SHA...$HEAD_SHA: ${{ steps.upload-diffs.outputs.artifact-url }}" >> comments
+ cat << EOF > comments
+ [{"body" : "test-suite diff from $BASE_SHA...$HEAD_SHA: ${{ steps.upload-diffs.outputs.artifact-url }}"}]
+ EOF
- name: Upload comment
uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
>From 399653cc81edd12e15d694cb945062f78136ab5b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 7 Apr 2026 15:42:27 +0800
Subject: [PATCH 22/30] Comment on run failure
---
.github/workflows/test-suite.yml | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 0f90c63e29ce7..310ef4f323a9e 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -116,6 +116,13 @@ jobs:
cat << EOF > comments
[{"body" : "test-suite diff from $BASE_SHA...$HEAD_SHA: ${{ steps.upload-diffs.outputs.artifact-url }}"}]
EOF
+ - name: Create comment on failure
+ if: failure()
+ run: |
+ RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ cat << EOF > comments
+ [{"body" : "Failed to get test-suite diff from $BASE_SHA...$HEAD_SHA: $RUN_URL"}]
+ EOF
- name: Upload comment
uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
>From c6d774f863085723787e9ef2e1ee025a43c8ea76 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 7 Apr 2026 17:35:17 +0800
Subject: [PATCH 23/30] Pass PR number into issue-write.yml
---
.github/workflows/issue-write.yml | 95 ++++++++++++++++---------------
.github/workflows/test-suite.yml | 7 ++-
2 files changed, 56 insertions(+), 46 deletions(-)
diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml
index 4de4e36dd3e03..f62e50ed24fd2 100644
--- a/.github/workflows/issue-write.yml
+++ b/.github/workflows/issue-write.yml
@@ -52,10 +52,14 @@ jobs:
script: |
var fs = require('fs');
var comments = []
+ var pr_number = 0
for (local_file of fs.readdirSync('.')) {
if (local_file.startsWith("comments")) {
comments.push(...JSON.parse(fs.readFileSync(local_file)))
}
+ if (local_file.startsWith("pr_number")) {
+ pr_number = parseInt(fs.readFileSync(local_file), 10)
+ }
}
if (!comments || comments.length == 0) {
return;
@@ -70,60 +74,61 @@ jobs:
console.log(runInfo);
- // Query to find the number of the pull request that triggered this job.
- // The associated pull requests are based off of the branch name, so if
- // you create a pull request for a branch, close it, and then create
- // another pull request with the same branch, then this query will return
- // two associated pull requests. This is why we have to fetch all the
- // associated pull requests and then iterate through them to find the
- // one that is open.
- const gql_query = `
- query($repo_owner : String!, $repo_name : String!, $branch: String!) {
- repository(owner: $repo_owner, name: $repo_name) {
- ref (qualifiedName: $branch) {
- associatedPullRequests(first: 100) {
- nodes {
- baseRepository {
- owner {
- login
+ if (!pr_number) {
+ // Query to find the number of the pull request that triggered this job.
+ // The associated pull requests are based off of the branch name, so if
+ // you create a pull request for a branch, close it, and then create
+ // another pull request with the same branch, then this query will return
+ // two associated pull requests. This is why we have to fetch all the
+ // associated pull requests and then iterate through them to find the
+ // one that is open.
+ const gql_query = `
+ query($repo_owner : String!, $repo_name : String!, $branch: String!) {
+ repository(owner: $repo_owner, name: $repo_name) {
+ ref (qualifiedName: $branch) {
+ associatedPullRequests(first: 100) {
+ nodes {
+ baseRepository {
+ owner {
+ login
+ }
}
+ number
+ state
}
- number
- state
}
}
}
}
+ `
+ const gql_variables = {
+ repo_owner: runInfo.data.head_repository.owner.login,
+ repo_name: runInfo.data.head_repository.name,
+ branch: runInfo.data.head_branch
}
- `
- const gql_variables = {
- repo_owner: runInfo.data.head_repository.owner.login,
- repo_name: runInfo.data.head_repository.name,
- branch: runInfo.data.head_branch
- }
- const gql_result = await github.graphql(gql_query, gql_variables);
- console.log(gql_result);
- // If the branch for the PR was deleted before this job has a chance
- // to run, then the ref will be null. This can happen if someone:
- // 1. Rebase the PR, which triggers some workflow.
- // 2. Immediately merges the PR and deletes the branch.
- // 3. The workflow finishes and triggers this job.
- if (!gql_result.repository.ref) {
- console.log("Ref has been deleted");
- return;
- }
- console.log(gql_result.repository.ref.associatedPullRequests.nodes);
+ const gql_result = await github.graphql(gql_query, gql_variables);
+ console.log(gql_result);
+ // If the branch for the PR was deleted before this job has a chance
+ // to run, then the ref will be null. This can happen if someone:
+ // 1. Rebase the PR, which triggers some workflow.
+ // 2. Immediately merges the PR and deletes the branch.
+ // 3. The workflow finishes and triggers this job.
+ if (!gql_result.repository.ref) {
+ console.log("Ref has been deleted");
+ return;
+ }
+ console.log(gql_result.repository.ref.associatedPullRequests.nodes);
- var pr_number = 0;
- gql_result.repository.ref.associatedPullRequests.nodes.forEach((pr) => {
+ gql_result.repository.ref.associatedPullRequests.nodes.forEach((pr) => {
- // The largest PR number is the one we care about. The only way
- // to have more than one associated pull requests is if all the
- // old pull requests are in the closed state.
- if (pr.baseRepository.owner.login = context.repo.owner && pr.number > pr_number) {
- pr_number = pr.number;
- }
- });
+ // The largest PR number is the one we care about. The only way
+ // to have more than one associated pull requests is if all the
+ // old pull requests are in the closed state.
+ if (pr.baseRepository.owner.login = context.repo.owner && pr.number > pr_number) {
+ pr_number = pr.number;
+ }
+ });
+ }
if (pr_number == 0) {
console.log("Error retrieving pull request number");
return;
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 310ef4f323a9e..638a662713563 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -39,6 +39,7 @@ jobs:
[{"body": "Unable to diff test-suite with PR, PR isn't mergeable"}]
EOF
exit 1
+ - run: exit 1
- name: Checkout pull request
uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
@@ -123,10 +124,14 @@ jobs:
cat << EOF > comments
[{"body" : "Failed to get test-suite diff from $BASE_SHA...$HEAD_SHA: $RUN_URL"}]
EOF
- - name: Upload comment
+ - name: Save PR number
+ if: always()
+ run: echo ${{ fromJSON(steps.get-pr.outputs.result).id }} > pr_number
+ - name: Upload comment and PR number
uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
with:
name: workflow-args
path: |
comments
+ pr_number
>From 3c33b26c1ae0cb0d38f87b0f8782405695233b39 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 7 Apr 2026 17:37:37 +0800
Subject: [PATCH 24/30] Use PR number not id
---
.github/workflows/test-suite.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 638a662713563..5185417c3ffbd 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -39,7 +39,6 @@ jobs:
[{"body": "Unable to diff test-suite with PR, PR isn't mergeable"}]
EOF
exit 1
- - run: exit 1
- name: Checkout pull request
uses: actions/checkout at 8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
@@ -126,7 +125,7 @@ jobs:
EOF
- name: Save PR number
if: always()
- run: echo ${{ fromJSON(steps.get-pr.outputs.result).id }} > pr_number
+ run: echo ${{ fromJSON(steps.get-pr.outputs.result).number }} > pr_number
- name: Upload comment and PR number
uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
>From f7ba7283527a8dae8179f3234f72e559faddd92e Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 7 Apr 2026 17:49:57 +0800
Subject: [PATCH 25/30] Fix LLVM_APPEND_VC_REV typo
---
.github/workflows/test-suite/llvm.cmake | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/test-suite/llvm.cmake b/.github/workflows/test-suite/llvm.cmake
index 20d21e7f3495d..e91dbba47bedb 100644
--- a/.github/workflows/test-suite/llvm.cmake
+++ b/.github/workflows/test-suite/llvm.cmake
@@ -1,4 +1,4 @@
set(CMAKE_BUILD_TYPE Release CACHE STRING "")
set(LLVM_TARGETS_TO_BUILD "AArch64;RISCV;X86" CACHE STRING "")
set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
-set(LLVM_APPEND_VC_REF OFF CACHE STRING "")
+set(LLVM_APPEND_VC_REV OFF CACHE BOOL "")
>From eeee3bc828c5ad4d4ee31aed02a0ed7deb6e735d Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 7 Apr 2026 21:49:02 +0800
Subject: [PATCH 26/30] Pin ci container image
---
.github/workflows/test-suite.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 5185417c3ffbd..d6a5b49c98f35 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -18,7 +18,7 @@ jobs:
name: Build test-suite and diff
runs-on: ubuntu-24.04
container:
- image: ghcr.io/llvm/ci-ubuntu-24.04:latest
+ image: ghcr.io/llvm/ci-ubuntu-24.04:35e958d7f0b7
if: github.event.issue.pull_request && startswith(github.event.comment.body, '/test-suite')
steps:
- name: Get pull request
>From cd8ffdfd9774337fa131009e2785fdb2166db788 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 7 Apr 2026 21:55:09 +0800
Subject: [PATCH 27/30] Address zizmor template expansion warnings
---
.github/workflows/test-suite.yml | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index d6a5b49c98f35..4fd5621b82948 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -118,14 +118,17 @@ jobs:
EOF
- name: Create comment on failure
if: failure()
+ env:
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
- RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
cat << EOF > comments
[{"body" : "Failed to get test-suite diff from $BASE_SHA...$HEAD_SHA: $RUN_URL"}]
EOF
- name: Save PR number
if: always()
- run: echo ${{ fromJSON(steps.get-pr.outputs.result).number }} > pr_number
+ env:
+ PR_NUMBER: ${{ fromJSON(steps.get-pr.outputs.result).number }}
+ run: echo $PR_NUMBER > pr_number
- name: Upload comment and PR number
uses: actions/upload-artifact at bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
>From 0521211c70c0e6edf14ad5290c6401330b90fd7f Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 16 Apr 2026 11:25:54 +0100
Subject: [PATCH 28/30] Strip out changes from #192205
---
.github/workflows/issue-write.yml | 98 +++++++++++++++----------------
1 file changed, 46 insertions(+), 52 deletions(-)
diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml
index f62e50ed24fd2..36c5877d87309 100644
--- a/.github/workflows/issue-write.yml
+++ b/.github/workflows/issue-write.yml
@@ -23,8 +23,7 @@ jobs:
permissions:
pull-requests: write
if: >
- (github.event.workflow_run.event == 'pull_request' ||
- github.event.workflow_run.event == 'issue_comment') &&
+ github.event.workflow_run.event == 'pull_request' &&
(
github.event.workflow_run.conclusion == 'success' ||
github.event.workflow_run.conclusion == 'failure'
@@ -52,14 +51,10 @@ jobs:
script: |
var fs = require('fs');
var comments = []
- var pr_number = 0
for (local_file of fs.readdirSync('.')) {
if (local_file.startsWith("comments")) {
comments.push(...JSON.parse(fs.readFileSync(local_file)))
}
- if (local_file.startsWith("pr_number")) {
- pr_number = parseInt(fs.readFileSync(local_file), 10)
- }
}
if (!comments || comments.length == 0) {
return;
@@ -74,61 +69,60 @@ jobs:
console.log(runInfo);
- if (!pr_number) {
- // Query to find the number of the pull request that triggered this job.
- // The associated pull requests are based off of the branch name, so if
- // you create a pull request for a branch, close it, and then create
- // another pull request with the same branch, then this query will return
- // two associated pull requests. This is why we have to fetch all the
- // associated pull requests and then iterate through them to find the
- // one that is open.
- const gql_query = `
- query($repo_owner : String!, $repo_name : String!, $branch: String!) {
- repository(owner: $repo_owner, name: $repo_name) {
- ref (qualifiedName: $branch) {
- associatedPullRequests(first: 100) {
- nodes {
- baseRepository {
- owner {
- login
- }
+ // Query to find the number of the pull request that triggered this job.
+ // The associated pull requests are based off of the branch name, so if
+ // you create a pull request for a branch, close it, and then create
+ // another pull request with the same branch, then this query will return
+ // two associated pull requests. This is why we have to fetch all the
+ // associated pull requests and then iterate through them to find the
+ // one that is open.
+ const gql_query = `
+ query($repo_owner : String!, $repo_name : String!, $branch: String!) {
+ repository(owner: $repo_owner, name: $repo_name) {
+ ref (qualifiedName: $branch) {
+ associatedPullRequests(first: 100) {
+ nodes {
+ baseRepository {
+ owner {
+ login
}
- number
- state
}
+ number
+ state
}
}
}
}
- `
- const gql_variables = {
- repo_owner: runInfo.data.head_repository.owner.login,
- repo_name: runInfo.data.head_repository.name,
- branch: runInfo.data.head_branch
- }
- const gql_result = await github.graphql(gql_query, gql_variables);
- console.log(gql_result);
- // If the branch for the PR was deleted before this job has a chance
- // to run, then the ref will be null. This can happen if someone:
- // 1. Rebase the PR, which triggers some workflow.
- // 2. Immediately merges the PR and deletes the branch.
- // 3. The workflow finishes and triggers this job.
- if (!gql_result.repository.ref) {
- console.log("Ref has been deleted");
- return;
}
- console.log(gql_result.repository.ref.associatedPullRequests.nodes);
+ `
+ const gql_variables = {
+ repo_owner: runInfo.data.head_repository.owner.login,
+ repo_name: runInfo.data.head_repository.name,
+ branch: runInfo.data.head_branch
+ }
+ const gql_result = await github.graphql(gql_query, gql_variables);
+ console.log(gql_result);
+ // If the branch for the PR was deleted before this job has a chance
+ // to run, then the ref will be null. This can happen if someone:
+ // 1. Rebase the PR, which triggers some workflow.
+ // 2. Immediately merges the PR and deletes the branch.
+ // 3. The workflow finishes and triggers this job.
+ if (!gql_result.repository.ref) {
+ console.log("Ref has been deleted");
+ return;
+ }
+ console.log(gql_result.repository.ref.associatedPullRequests.nodes);
- gql_result.repository.ref.associatedPullRequests.nodes.forEach((pr) => {
+ var pr_number = 0;
+ gql_result.repository.ref.associatedPullRequests.nodes.forEach((pr) => {
- // The largest PR number is the one we care about. The only way
- // to have more than one associated pull requests is if all the
- // old pull requests are in the closed state.
- if (pr.baseRepository.owner.login = context.repo.owner && pr.number > pr_number) {
- pr_number = pr.number;
- }
- });
- }
+ // The largest PR number is the one we care about. The only way
+ // to have more than one associated pull requests is if all the
+ // old pull requests are in the closed state.
+ if (pr.baseRepository.owner.login = context.repo.owner && pr.number > pr_number) {
+ pr_number = pr.number;
+ }
+ });
if (pr_number == 0) {
console.log("Error retrieving pull request number");
return;
>From 9aa79302060b9e51ba97b733d6ff210e0d12d614 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 16 Apr 2026 11:26:14 +0100
Subject: [PATCH 29/30] Add TODO to remove cmake install step
---
.github/workflows/test-suite.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index 4fd5621b82948..985210c9f409e 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -65,6 +65,7 @@ jobs:
- name: Install system dependencies
run: |
# Install newer version of CMake (llvm/ci-ubuntu-24.04 image has CMake 3.28)
+ # TODO(boomanaiden154): Remove once upgraded to Ubuntu 26.04
sudo apt-get update
sudo apt-get install -y wget
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
>From a37a98ec9d90ca8e6a501520cd4cce7427ef5077 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 16 Apr 2026 11:34:47 +0100
Subject: [PATCH 30/30] sccache llvm build
---
.github/workflows/test-suite/llvm.cmake | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/test-suite/llvm.cmake b/.github/workflows/test-suite/llvm.cmake
index e91dbba47bedb..ae7ef254eb524 100644
--- a/.github/workflows/test-suite/llvm.cmake
+++ b/.github/workflows/test-suite/llvm.cmake
@@ -2,3 +2,5 @@ set(CMAKE_BUILD_TYPE Release CACHE STRING "")
set(LLVM_TARGETS_TO_BUILD "AArch64;RISCV;X86" CACHE STRING "")
set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
set(LLVM_APPEND_VC_REV OFF CACHE BOOL "")
+set(CMAKE_C_COMPILER_LAUNCHER sccache CACHE STRING "")
+set(CMAKE_CXX_COMPILER_LAUNCHER sccache CACHE STRING "")
More information about the llvm-commits
mailing list