[libcxx] [llvm] [libc++] Add a job that runs the benchmarks as part of the CI (PR #157567)
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 11 11:45:42 PDT 2025
https://github.com/ldionne updated https://github.com/llvm/llvm-project/pull/157567
>From c7306f453076f0c8a962281ed406aad6ce6e21eb Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Mon, 8 Sep 2025 18:12:17 -0400
Subject: [PATCH 01/17] [libc++] Add a job that runs the benchmarks as part of
the CI
This will make it easier to see the performance impact of individual
patches. This is a first step towards better performance tracking, but
many things still need to be figured out:
- We should get a dedicated set of machines to run these tests on
- We could probably reuse regularly-generated baseline results published
to LNT instead of re-generating the baselines in this job every time.
---
.github/workflows/libcxx-build-and-test.yaml | 22 ++++++++++++++++++++
libcxx/utils/ci/run-buildbot | 18 ++++++++++++++++
2 files changed, 40 insertions(+)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 2e6ff7f91b6fc..80da80bdc59e7 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -284,3 +284,25 @@ jobs:
- name: Build and test
run: |
bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+
+ benchmarks:
+ runs-on: llvm-premerge-libcxx-next-runners # TODO: This should run on a dedicated set of machines
+ # needs: [ stage3 ] # TODO: Run this at the end of the pipeline once we're done testing the job
+ steps:
+ - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ - name: benchmarks
+ run: libcxx/utils/ci/run-buildbot benchmark-comparison
+ env:
+ CC: clang-22
+ CXX: clang++-22
+ - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+ if: always()
+ with:
+ name: benchmarks-results
+ path: |
+ **/test-results.xml
+ **/*.abilist
+ **/CMakeConfigureLog.yaml
+ **/CMakeError.log
+ **/CMakeOutput.log
+ **/crash_diagnostics/*
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 57ecf1e49dbf2..6a776b6645723 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -542,6 +542,24 @@ generic-optimized-speed)
generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-optimized-speed.cmake"
check-runtimes
;;
+benchmark-comparison)
+ clean
+ LIT_ARGS="-sv -j1 --param optimization=speed libcxx/test/benchmarks/hash.bench.cpp libcxx/test/benchmarks/join_view.bench.cpp"
+
+ baseline_commit=$(git merge-base main HEAD)
+ step "Building and running the baseline at ${baseline_commit}"
+ rm -rf "${BUILD_DIR}-baseline"
+ ./libcxx/utils/test-at-commit --commit ${baseline_commit} -B "${BUILD_DIR}-baseline" -- ${LIT_ARGS}
+
+ # Build and run the candidate
+ step "Building and running the candidate"
+ generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-optimized-speed.cmake"
+ ./libcxx/utils/libcxx-lit "${BUILD_DIR}" ${LIT_ARGS}
+
+ step "Comparison of the baseline and the candidate"
+ ./libcxx/utils/compare-benchmarks <(./libcxx/utils/consolidate-benchmarks "${BUILD_DIR}-baseline") \
+ <(./libcxx/utils/consolidate-benchmarks "${BUILD_DIR}")
+;;
apple-configuration)
clean
>From 567efcabb14f4b695a9b267b505443d0d592c05b Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 12:25:40 -0400
Subject: [PATCH 02/17] Try to fix baseline commit
---
libcxx/utils/ci/run-buildbot | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 6a776b6645723..b888faed975da 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -546,10 +546,12 @@ benchmark-comparison)
clean
LIT_ARGS="-sv -j1 --param optimization=speed libcxx/test/benchmarks/hash.bench.cpp libcxx/test/benchmarks/join_view.bench.cpp"
- baseline_commit=$(git merge-base main HEAD)
- step "Building and running the baseline at ${baseline_commit}"
+ if [[ "${BASELINE_COMMIT}" == "" ]]; then
+ BASELINE_COMMIT=$(git merge-base main HEAD)
+ fi
+ step "Building and running the baseline at ${BASELINE_COMMIT}"
rm -rf "${BUILD_DIR}-baseline"
- ./libcxx/utils/test-at-commit --commit ${baseline_commit} -B "${BUILD_DIR}-baseline" -- ${LIT_ARGS}
+ ./libcxx/utils/test-at-commit --commit ${BASELINE_COMMIT} -B "${BUILD_DIR}-baseline" -- ${LIT_ARGS}
# Build and run the candidate
step "Building and running the candidate"
>From 9ac9dc92978a281f2e0d0e652aca74250be2858d Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 12:31:40 -0400
Subject: [PATCH 03/17] Try again
---
libcxx/utils/ci/run-buildbot | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index b888faed975da..373e298183d86 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -546,12 +546,15 @@ benchmark-comparison)
clean
LIT_ARGS="-sv -j1 --param optimization=speed libcxx/test/benchmarks/hash.bench.cpp libcxx/test/benchmarks/join_view.bench.cpp"
- if [[ "${BASELINE_COMMIT}" == "" ]]; then
- BASELINE_COMMIT=$(git merge-base main HEAD)
+ if [[ "${GITHUB_BASE_REF}" != "" ]]; then # triggered from a PR
+ baseline_commit=$(git merge-base ${GITHUB_BASE_REF} ${GITHUB_SHA})
+ else
+ baseline_commit=$(git merge-base main HEAD)
fi
- step "Building and running the baseline at ${BASELINE_COMMIT}"
+
+ step "Building and running the baseline at ${baseline_commit}"
rm -rf "${BUILD_DIR}-baseline"
- ./libcxx/utils/test-at-commit --commit ${BASELINE_COMMIT} -B "${BUILD_DIR}-baseline" -- ${LIT_ARGS}
+ ./libcxx/utils/test-at-commit --commit ${baseline_commit} -B "${BUILD_DIR}-baseline" -- ${LIT_ARGS}
# Build and run the candidate
step "Building and running the candidate"
>From f16b1a7988c3f0e90a75a3fce868f11d30672292 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 12:45:04 -0400
Subject: [PATCH 04/17] Try to fix the fetch
---
.github/workflows/libcxx-build-and-test.yaml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 80da80bdc59e7..73e01611025ef 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -290,6 +290,8 @@ jobs:
# needs: [ stage3 ] # TODO: Run this at the end of the pipeline once we're done testing the job
steps:
- uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ with:
+ fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
- name: benchmarks
run: libcxx/utils/ci/run-buildbot benchmark-comparison
env:
>From 354e90385aaaa20031a10b47b1111012823c1284 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 13:05:07 -0400
Subject: [PATCH 05/17] Try fetching tags
---
.github/workflows/libcxx-build-and-test.yaml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 73e01611025ef..cd125e7e1a52f 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -292,6 +292,7 @@ jobs:
- uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
+ fetch-tags: true
- name: benchmarks
run: libcxx/utils/ci/run-buildbot benchmark-comparison
env:
>From 716aa962b46afcd9926de8f4818bf4c4d662addf Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 13:15:59 -0400
Subject: [PATCH 06/17] Try to fix merge base
---
libcxx/utils/ci/run-buildbot | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 373e298183d86..1b3d1a7861c48 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -547,7 +547,7 @@ benchmark-comparison)
LIT_ARGS="-sv -j1 --param optimization=speed libcxx/test/benchmarks/hash.bench.cpp libcxx/test/benchmarks/join_view.bench.cpp"
if [[ "${GITHUB_BASE_REF}" != "" ]]; then # triggered from a PR
- baseline_commit=$(git merge-base ${GITHUB_BASE_REF} ${GITHUB_SHA})
+ baseline_commit=$(git merge-base refs/heads/${GITHUB_BASE_REF} ${GITHUB_SHA})
else
baseline_commit=$(git merge-base main HEAD)
fi
>From a9b70dfc099e95cb838b78c05e83fc8720bb3e91 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 13:26:57 -0400
Subject: [PATCH 07/17] Try checkout main
---
.github/workflows/libcxx-build-and-test.yaml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index cd125e7e1a52f..6ede9a312e443 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -293,6 +293,7 @@ jobs:
with:
fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
fetch-tags: true
+ ref: main
- name: benchmarks
run: libcxx/utils/ci/run-buildbot benchmark-comparison
env:
>From dcc96726f024caa13f6334bcae0b2f63af8b146b Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 13:39:00 -0400
Subject: [PATCH 08/17] Try to fix checkout
---
.github/workflows/libcxx-build-and-test.yaml | 2 +-
libcxx/utils/ci/run-buildbot | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 6ede9a312e443..b30e126181a83 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -293,7 +293,7 @@ jobs:
with:
fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
fetch-tags: true
- ref: main
+ # ref: main
- name: benchmarks
run: libcxx/utils/ci/run-buildbot benchmark-comparison
env:
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 1b3d1a7861c48..3daf524503710 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -547,7 +547,7 @@ benchmark-comparison)
LIT_ARGS="-sv -j1 --param optimization=speed libcxx/test/benchmarks/hash.bench.cpp libcxx/test/benchmarks/join_view.bench.cpp"
if [[ "${GITHUB_BASE_REF}" != "" ]]; then # triggered from a PR
- baseline_commit=$(git merge-base refs/heads/${GITHUB_BASE_REF} ${GITHUB_SHA})
+ baseline_commit=$(git merge-base refs/remotes/origin/${GITHUB_BASE_REF} ${GITHUB_SHA})
else
baseline_commit=$(git merge-base main HEAD)
fi
>From 900f6b520de8746db135db7eff4bfcb2b78edaae Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 13:48:18 -0400
Subject: [PATCH 09/17] Install python requirements'
---
.github/workflows/libcxx-build-and-test.yaml | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index b30e126181a83..7cb5c0609d56d 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -293,9 +293,12 @@ jobs:
with:
fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
fetch-tags: true
- # ref: main
- name: benchmarks
- run: libcxx/utils/ci/run-buildbot benchmark-comparison
+ run: |
+ python3 -m venv .venv
+ source .venv/bin/activate
+ python -m pip install -r libcxx/utils/requirements.txt
+ libcxx/utils/ci/run-buildbot benchmark-comparison
env:
CC: clang-22
CXX: clang++-22
>From 38eab49b1af376b7b76379a47f1fc87c07e97dc2 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 13:55:51 -0400
Subject: [PATCH 10/17] Try to fix issue with venv
---
.github/workflows/libcxx-build-and-test.yaml | 2 ++
libcxx/utils/ci/Dockerfile | 1 +
2 files changed, 3 insertions(+)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 7cb5c0609d56d..1ea99abaa7c8e 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -294,7 +294,9 @@ jobs:
fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
fetch-tags: true
- name: benchmarks
+ # TODO: python3-venv should be taken from the Docker image instead
run: |
+ sudo apt-get install -y python3-venv
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r libcxx/utils/requirements.txt
diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile
index 79e11569c0d08..03c830499a0fd 100644
--- a/libcxx/utils/ci/Dockerfile
+++ b/libcxx/utils/ci/Dockerfile
@@ -100,6 +100,7 @@ RUN sudo apt-get update \
python3-packaging \
python3-setuptools \
python3-psutil \
+ python3-venv \
software-properties-common \
swig \
unzip \
>From 28cfc5b8f21492208bb5d430f893f993c36bf7c5 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 14:02:49 -0400
Subject: [PATCH 11/17] Fix package naem
---
.github/workflows/libcxx-build-and-test.yaml | 4 ++--
libcxx/utils/ci/Dockerfile | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 1ea99abaa7c8e..f4660e3c0c0d6 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -294,9 +294,9 @@ jobs:
fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
fetch-tags: true
- name: benchmarks
- # TODO: python3-venv should be taken from the Docker image instead
+ # TODO: python3-virtualenv should be taken from the Docker image instead
run: |
- sudo apt-get install -y python3-venv
+ sudo apt-get install -y python3-virtualenv
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r libcxx/utils/requirements.txt
diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile
index 03c830499a0fd..685f060b29f57 100644
--- a/libcxx/utils/ci/Dockerfile
+++ b/libcxx/utils/ci/Dockerfile
@@ -100,7 +100,7 @@ RUN sudo apt-get update \
python3-packaging \
python3-setuptools \
python3-psutil \
- python3-venv \
+ python3-virtualenv \
software-properties-common \
swig \
unzip \
>From 32c669d3e2ee5eecc09697fa5daf340a24cabcf2 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 14:03:25 -0400
Subject: [PATCH 12/17] Temporarily remove other jobs to reduce churn
---
.github/workflows/libcxx-build-and-test.yaml | 496 +++++++++----------
1 file changed, 248 insertions(+), 248 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index f4660e3c0c0d6..52d5551ae0fdd 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -34,256 +34,256 @@ concurrency:
cancel-in-progress: true
jobs:
- stage1:
- if: github.repository_owner == 'llvm'
- runs-on: llvm-premerge-libcxx-next-runners
- continue-on-error: false
- strategy:
- fail-fast: false
- matrix:
- config: [
- 'frozen-cxx03-headers',
- 'generic-cxx03',
- 'generic-cxx26',
- 'generic-modules'
- ]
- cc: [ 'clang-22' ]
- cxx: [ 'clang++-22' ]
- include:
- - config: 'generic-gcc'
- cc: 'gcc-15'
- cxx: 'g++-15'
- steps:
- - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- - name: ${{ matrix.config }}.${{ matrix.cxx }}
- run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
- env:
- CC: ${{ matrix.cc }}
- CXX: ${{ matrix.cxx }}
- - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
- if: always()
- with:
- name: ${{ matrix.config }}-${{ matrix.cxx }}-results
- path: |
- **/test-results.xml
- **/*.abilist
- **/CMakeConfigureLog.yaml
- **/CMakeError.log
- **/CMakeOutput.log
- **/crash_diagnostics/*
- stage2:
- if: github.repository_owner == 'llvm'
- runs-on: llvm-premerge-libcxx-next-runners
- needs: [ stage1 ]
- continue-on-error: false
- strategy:
- fail-fast: false
- matrix:
- config: [
- 'generic-cxx11',
- 'generic-cxx14',
- 'generic-cxx17',
- 'generic-cxx20',
- 'generic-cxx23'
- ]
- cc: [ 'clang-22' ]
- cxx: [ 'clang++-22' ]
- include:
- - config: 'generic-gcc-cxx11'
- cc: 'gcc-15'
- cxx: 'g++-15'
- - config: 'generic-cxx26'
- cc: 'clang-21'
- cxx: 'clang++-21'
- - config: 'generic-cxx26'
- cc: 'clang-20'
- cxx: 'clang++-20'
- steps:
- - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- - name: ${{ matrix.config }}
- run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
- env:
- CC: ${{ matrix.cc }}
- CXX: ${{ matrix.cxx }}
- - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
- if: always() # Upload artifacts even if the build or test suite fails
- with:
- name: ${{ matrix.config }}-${{ matrix.cxx }}-results
- path: |
- **/test-results.xml
- **/*.abilist
- **/CMakeConfigureLog.yaml
- **/CMakeError.log
- **/CMakeOutput.log
- **/crash_diagnostics/*
- stage3:
- if: github.repository_owner == 'llvm'
- needs: [ stage2 ]
- continue-on-error: false
- strategy:
- fail-fast: false
- max-parallel: 8
- matrix:
- config: [
- 'generic-abi-unstable',
- 'generic-hardening-mode-debug',
- 'generic-hardening-mode-extensive',
- 'generic-hardening-mode-extensive-observe-semantic',
- 'generic-hardening-mode-fast',
- 'generic-hardening-mode-fast-with-abi-breaks',
- 'generic-merged',
- 'generic-modules-cxx17-lsv',
- 'generic-no-exceptions',
- 'generic-no-experimental',
- 'generic-no-filesystem',
- 'generic-no-localization',
- 'generic-no-terminal',
- 'generic-no-random_device',
- 'generic-no-threads',
- 'generic-no-tzdb',
- 'generic-no-unicode',
- 'generic-no-wide-characters',
- 'generic-no-rtti',
- 'generic-optimized-speed',
- 'generic-static',
- 'bootstrapping-build'
- ]
- machine: [ 'llvm-premerge-libcxx-next-runners' ]
- include:
- - config: 'generic-cxx26'
- machine: llvm-premerge-libcxx-next-runners
- - config: 'generic-asan'
- machine: llvm-premerge-libcxx-next-runners
- - config: 'generic-tsan'
- machine: llvm-premerge-libcxx-next-runners
- - config: 'generic-ubsan'
- machine: llvm-premerge-libcxx-next-runners
- # Use a larger machine for MSAN to avoid timeout and memory allocation issues.
- - config: 'generic-msan'
- machine: llvm-premerge-libcxx-next-runners
- runs-on: ${{ matrix.machine }}
- steps:
- - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- - name: ${{ matrix.config }}
- run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
- env:
- CC: clang-22
- CXX: clang++-22
- - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
- if: always()
- with:
- name: ${{ matrix.config }}-results
- path: |
- **/test-results.xml
- **/*.abilist
- **/CMakeConfigureLog.yaml
- **/CMakeError.log
- **/CMakeOutput.log
- **/crash_diagnostics/*
+ # stage1:
+ # if: github.repository_owner == 'llvm'
+ # runs-on: llvm-premerge-libcxx-next-runners
+ # continue-on-error: false
+ # strategy:
+ # fail-fast: false
+ # matrix:
+ # config: [
+ # 'frozen-cxx03-headers',
+ # 'generic-cxx03',
+ # 'generic-cxx26',
+ # 'generic-modules'
+ # ]
+ # cc: [ 'clang-22' ]
+ # cxx: [ 'clang++-22' ]
+ # include:
+ # - config: 'generic-gcc'
+ # cc: 'gcc-15'
+ # cxx: 'g++-15'
+ # steps:
+ # - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ # - name: ${{ matrix.config }}.${{ matrix.cxx }}
+ # run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+ # env:
+ # CC: ${{ matrix.cc }}
+ # CXX: ${{ matrix.cxx }}
+ # - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+ # if: always()
+ # with:
+ # name: ${{ matrix.config }}-${{ matrix.cxx }}-results
+ # path: |
+ # **/test-results.xml
+ # **/*.abilist
+ # **/CMakeConfigureLog.yaml
+ # **/CMakeError.log
+ # **/CMakeOutput.log
+ # **/crash_diagnostics/*
+ # stage2:
+ # if: github.repository_owner == 'llvm'
+ # runs-on: llvm-premerge-libcxx-next-runners
+ # needs: [ stage1 ]
+ # continue-on-error: false
+ # strategy:
+ # fail-fast: false
+ # matrix:
+ # config: [
+ # 'generic-cxx11',
+ # 'generic-cxx14',
+ # 'generic-cxx17',
+ # 'generic-cxx20',
+ # 'generic-cxx23'
+ # ]
+ # cc: [ 'clang-22' ]
+ # cxx: [ 'clang++-22' ]
+ # include:
+ # - config: 'generic-gcc-cxx11'
+ # cc: 'gcc-15'
+ # cxx: 'g++-15'
+ # - config: 'generic-cxx26'
+ # cc: 'clang-21'
+ # cxx: 'clang++-21'
+ # - config: 'generic-cxx26'
+ # cc: 'clang-20'
+ # cxx: 'clang++-20'
+ # steps:
+ # - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ # - name: ${{ matrix.config }}
+ # run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+ # env:
+ # CC: ${{ matrix.cc }}
+ # CXX: ${{ matrix.cxx }}
+ # - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+ # if: always() # Upload artifacts even if the build or test suite fails
+ # with:
+ # name: ${{ matrix.config }}-${{ matrix.cxx }}-results
+ # path: |
+ # **/test-results.xml
+ # **/*.abilist
+ # **/CMakeConfigureLog.yaml
+ # **/CMakeError.log
+ # **/CMakeOutput.log
+ # **/crash_diagnostics/*
+ # stage3:
+ # if: github.repository_owner == 'llvm'
+ # needs: [ stage2 ]
+ # continue-on-error: false
+ # strategy:
+ # fail-fast: false
+ # max-parallel: 8
+ # matrix:
+ # config: [
+ # 'generic-abi-unstable',
+ # 'generic-hardening-mode-debug',
+ # 'generic-hardening-mode-extensive',
+ # 'generic-hardening-mode-extensive-observe-semantic',
+ # 'generic-hardening-mode-fast',
+ # 'generic-hardening-mode-fast-with-abi-breaks',
+ # 'generic-merged',
+ # 'generic-modules-cxx17-lsv',
+ # 'generic-no-exceptions',
+ # 'generic-no-experimental',
+ # 'generic-no-filesystem',
+ # 'generic-no-localization',
+ # 'generic-no-terminal',
+ # 'generic-no-random_device',
+ # 'generic-no-threads',
+ # 'generic-no-tzdb',
+ # 'generic-no-unicode',
+ # 'generic-no-wide-characters',
+ # 'generic-no-rtti',
+ # 'generic-optimized-speed',
+ # 'generic-static',
+ # 'bootstrapping-build'
+ # ]
+ # machine: [ 'llvm-premerge-libcxx-next-runners' ]
+ # include:
+ # - config: 'generic-cxx26'
+ # machine: llvm-premerge-libcxx-next-runners
+ # - config: 'generic-asan'
+ # machine: llvm-premerge-libcxx-next-runners
+ # - config: 'generic-tsan'
+ # machine: llvm-premerge-libcxx-next-runners
+ # - config: 'generic-ubsan'
+ # machine: llvm-premerge-libcxx-next-runners
+ # # Use a larger machine for MSAN to avoid timeout and memory allocation issues.
+ # - config: 'generic-msan'
+ # machine: llvm-premerge-libcxx-next-runners
+ # runs-on: ${{ matrix.machine }}
+ # steps:
+ # - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ # - name: ${{ matrix.config }}
+ # run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+ # env:
+ # CC: clang-22
+ # CXX: clang++-22
+ # - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+ # if: always()
+ # with:
+ # name: ${{ matrix.config }}-results
+ # path: |
+ # **/test-results.xml
+ # **/*.abilist
+ # **/CMakeConfigureLog.yaml
+ # **/CMakeError.log
+ # **/CMakeOutput.log
+ # **/crash_diagnostics/*
- macos:
- needs: [ stage2 ]
- strategy:
- fail-fast: false
- matrix:
- include:
- - config: generic-cxx03
- os: macos-15
- - config: generic-cxx23
- os: macos-15
- - config: generic-modules
- os: macos-15
- - config: apple-configuration
- os: macos-15
- # TODO: These jobs are intended to test back-deployment (building against ToT libc++ but running against an
- # older system-provided libc++.dylib). Doing this properly would require building the test suite on a
- # recent macOS using a recent Clang (hence recent Xcode), and then running the actual test suite on an
- # older mac. We could do that by e.g. sharing artifacts between the two jobs.
- #
- # However, our Lit configuration currently doesn't provide a good way to do that in a batch, so our only
- # alternative is to actually build on the same host that we're going to run on. Sadly, that doesn't work
- # since older macOSes don't support newer Xcodes. For now, we run the "backdeployment" jobs on recent
- # macOS versions as a way to avoid rotting that configuration, but it doesn't provide a lot of additional
- # coverage.
- - config: apple-system
- os: macos-15
- - config: apple-system-hardened
- os: macos-15
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- - uses: maxim-lobanov/setup-xcode at 60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
- with:
- # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md
- xcode-version: '16.3'
- - uses: seanmiddleditch/gha-setup-ninja at 3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6
- - name: Build and test
- run: |
- python3 -m venv .venv
- source .venv/bin/activate
- python -m pip install psutil
- bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
- - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
- if: always() # Upload artifacts even if the build or test suite fails
- with:
- name: macos-${{ matrix.config }}-results
- path: |
- **/test-results.xml
- **/*.abilist
- **/CMakeConfigureLog.yaml
- **/CMakeError.log
- **/CMakeOutput.log
- **/crash_diagnostics/*
+ # macos:
+ # needs: [ stage2 ]
+ # strategy:
+ # fail-fast: false
+ # matrix:
+ # include:
+ # - config: generic-cxx03
+ # os: macos-15
+ # - config: generic-cxx23
+ # os: macos-15
+ # - config: generic-modules
+ # os: macos-15
+ # - config: apple-configuration
+ # os: macos-15
+ # # TODO: These jobs are intended to test back-deployment (building against ToT libc++ but running against an
+ # # older system-provided libc++.dylib). Doing this properly would require building the test suite on a
+ # # recent macOS using a recent Clang (hence recent Xcode), and then running the actual test suite on an
+ # # older mac. We could do that by e.g. sharing artifacts between the two jobs.
+ # #
+ # # However, our Lit configuration currently doesn't provide a good way to do that in a batch, so our only
+ # # alternative is to actually build on the same host that we're going to run on. Sadly, that doesn't work
+ # # since older macOSes don't support newer Xcodes. For now, we run the "backdeployment" jobs on recent
+ # # macOS versions as a way to avoid rotting that configuration, but it doesn't provide a lot of additional
+ # # coverage.
+ # - config: apple-system
+ # os: macos-15
+ # - config: apple-system-hardened
+ # os: macos-15
+ # runs-on: ${{ matrix.os }}
+ # steps:
+ # - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ # - uses: maxim-lobanov/setup-xcode at 60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
+ # with:
+ # # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md
+ # xcode-version: '16.3'
+ # - uses: seanmiddleditch/gha-setup-ninja at 3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6
+ # - name: Build and test
+ # run: |
+ # python3 -m venv .venv
+ # source .venv/bin/activate
+ # python -m pip install psutil
+ # bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+ # - uses: actions/upload-artifact at 26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+ # if: always() # Upload artifacts even if the build or test suite fails
+ # with:
+ # name: macos-${{ matrix.config }}-results
+ # path: |
+ # **/test-results.xml
+ # **/*.abilist
+ # **/CMakeConfigureLog.yaml
+ # **/CMakeError.log
+ # **/CMakeOutput.log
+ # **/crash_diagnostics/*
- windows:
- runs-on: windows-2022
- needs: [ stage2 ]
- strategy:
- fail-fast: false
- matrix:
- include:
- - { config: clang-cl-dll, mingw: false }
- - { config: clang-cl-static, mingw: false }
- - { config: clang-cl-no-vcruntime, mingw: false }
- - { config: clang-cl-debug, mingw: false }
- - { config: clang-cl-static-crt, mingw: false }
- - { config: mingw-dll, mingw: true }
- - { config: mingw-static, mingw: true }
- - { config: mingw-dll-i686, mingw: true }
- - { config: mingw-incomplete-sysroot, mingw: true }
- steps:
- - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- - name: Install dependencies
- run: |
- choco install -y ninja
- pip install psutil
- - name: Install a current LLVM
- if: ${{ matrix.mingw != true }}
- run: |
- choco install -y llvm --version=20.1.8 --allow-downgrade
- - name: Install llvm-mingw
- if: ${{ matrix.mingw == true }}
- run: |
- curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20250709/llvm-mingw-20250709-ucrt-x86_64.zip
- powershell Expand-Archive llvm-mingw*.zip -DestinationPath .
- del llvm-mingw*.zip
- mv llvm-mingw* c:\llvm-mingw
- echo "c:\llvm-mingw\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
- - name: Simulate a from-scratch build of llvm-mingw
- if: ${{ matrix.config == 'mingw-incomplete-sysroot' }}
- run: |
- rm -r c:\llvm-mingw\include\c++
- rm -r c:\llvm-mingw\*-w64-mingw32\lib\libc++*
- rm -r c:\llvm-mingw\*-w64-mingw32\lib\libunwind*
- - name: Add Git Bash to the path
- run: |
- echo "c:\Program Files\Git\usr\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
- - name: Set up the MSVC dev environment
- if: ${{ matrix.mingw != true }}
- uses: ilammy/msvc-dev-cmd at 0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
- - name: Build and test
- run: |
- bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+ # windows:
+ # runs-on: windows-2022
+ # needs: [ stage2 ]
+ # strategy:
+ # fail-fast: false
+ # matrix:
+ # include:
+ # - { config: clang-cl-dll, mingw: false }
+ # - { config: clang-cl-static, mingw: false }
+ # - { config: clang-cl-no-vcruntime, mingw: false }
+ # - { config: clang-cl-debug, mingw: false }
+ # - { config: clang-cl-static-crt, mingw: false }
+ # - { config: mingw-dll, mingw: true }
+ # - { config: mingw-static, mingw: true }
+ # - { config: mingw-dll-i686, mingw: true }
+ # - { config: mingw-incomplete-sysroot, mingw: true }
+ # steps:
+ # - uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ # - name: Install dependencies
+ # run: |
+ # choco install -y ninja
+ # pip install psutil
+ # - name: Install a current LLVM
+ # if: ${{ matrix.mingw != true }}
+ # run: |
+ # choco install -y llvm --version=20.1.8 --allow-downgrade
+ # - name: Install llvm-mingw
+ # if: ${{ matrix.mingw == true }}
+ # run: |
+ # curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20250709/llvm-mingw-20250709-ucrt-x86_64.zip
+ # powershell Expand-Archive llvm-mingw*.zip -DestinationPath .
+ # del llvm-mingw*.zip
+ # mv llvm-mingw* c:\llvm-mingw
+ # echo "c:\llvm-mingw\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
+ # - name: Simulate a from-scratch build of llvm-mingw
+ # if: ${{ matrix.config == 'mingw-incomplete-sysroot' }}
+ # run: |
+ # rm -r c:\llvm-mingw\include\c++
+ # rm -r c:\llvm-mingw\*-w64-mingw32\lib\libc++*
+ # rm -r c:\llvm-mingw\*-w64-mingw32\lib\libunwind*
+ # - name: Add Git Bash to the path
+ # run: |
+ # echo "c:\Program Files\Git\usr\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
+ # - name: Set up the MSVC dev environment
+ # if: ${{ matrix.mingw != true }}
+ # uses: ilammy/msvc-dev-cmd at 0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
+ # - name: Build and test
+ # run: |
+ # bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
benchmarks:
runs-on: llvm-premerge-libcxx-next-runners # TODO: This should run on a dedicated set of machines
>From 20411483bcb4324d8dde0178513460117f8ea7f6 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 14:10:41 -0400
Subject: [PATCH 13/17] Try again
---
.github/workflows/libcxx-build-and-test.yaml | 2 +-
libcxx/utils/ci/Dockerfile | 1 -
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 52d5551ae0fdd..0a7e1c7afa566 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -296,7 +296,7 @@ jobs:
- name: benchmarks
# TODO: python3-virtualenv should be taken from the Docker image instead
run: |
- sudo apt-get install -y python3-virtualenv
+ sudo pip install virtualenv
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r libcxx/utils/requirements.txt
diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile
index 685f060b29f57..79e11569c0d08 100644
--- a/libcxx/utils/ci/Dockerfile
+++ b/libcxx/utils/ci/Dockerfile
@@ -100,7 +100,6 @@ RUN sudo apt-get update \
python3-packaging \
python3-setuptools \
python3-psutil \
- python3-virtualenv \
software-properties-common \
swig \
unzip \
>From 9742fe0d92acca52ab5596071b65e29ee4f75cdb Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 14:20:42 -0400
Subject: [PATCH 14/17] Try fixing
---
.github/workflows/libcxx-build-and-test.yaml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 0a7e1c7afa566..eed972ac6aa00 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -290,17 +290,17 @@ jobs:
# needs: [ stage3 ] # TODO: Run this at the end of the pipeline once we're done testing the job
steps:
- uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- with:
- fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
- fetch-tags: true
+ # with:
+ # fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
+ # fetch-tags: true
- name: benchmarks
# TODO: python3-virtualenv should be taken from the Docker image instead
+ # libcxx/utils/ci/run-buildbot benchmark-comparison
run: |
- sudo pip install virtualenv
+ python3 -m pip install virtualenv
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r libcxx/utils/requirements.txt
- libcxx/utils/ci/run-buildbot benchmark-comparison
env:
CC: clang-22
CXX: clang++-22
>From 43b799c21b75759d29a1181eff80ef8359c5f5a5 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 14:28:09 -0400
Subject: [PATCH 15/17] This is surprisingly fiddly to get working
---
.github/workflows/libcxx-build-and-test.yaml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index eed972ac6aa00..d829e293a269d 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -293,11 +293,13 @@ jobs:
# with:
# fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
# fetch-tags: true
+ - uses: actions/setup-python at v6
+ with:
+ python-version: '3.10'
- name: benchmarks
# TODO: python3-virtualenv should be taken from the Docker image instead
# libcxx/utils/ci/run-buildbot benchmark-comparison
run: |
- python3 -m pip install virtualenv
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r libcxx/utils/requirements.txt
>From a0a4026c5748160c87ff28ca98511953e0ffeb6a Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 14:40:59 -0400
Subject: [PATCH 16/17] Progress
---
.github/workflows/libcxx-build-and-test.yaml | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index d829e293a269d..6c18cd2a5fdf7 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -290,19 +290,18 @@ jobs:
# needs: [ stage3 ] # TODO: Run this at the end of the pipeline once we're done testing the job
steps:
- uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- # with:
- # fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
- # fetch-tags: true
+ with:
+ fetch-depth: 10
+ fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main
- uses: actions/setup-python at v6
with:
python-version: '3.10'
- name: benchmarks
- # TODO: python3-virtualenv should be taken from the Docker image instead
- # libcxx/utils/ci/run-buildbot benchmark-comparison
run: |
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r libcxx/utils/requirements.txt
+ libcxx/utils/ci/run-buildbot benchmark-comparison
env:
CC: clang-22
CXX: clang++-22
>From a7ebd66af17f89ccead596dd4fb8278597f29ec9 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 11 Sep 2025 14:45:24 -0400
Subject: [PATCH 17/17] Fetch all
---
.github/workflows/libcxx-build-and-test.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 6c18cd2a5fdf7..56461c6e8a235 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -291,7 +291,7 @@ jobs:
steps:
- uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
- fetch-depth: 10
+ fetch-depth: 0
fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main
- uses: actions/setup-python at v6
with:
More information about the llvm-commits
mailing list