[llvm] workflows: Rewrite build-ci-container to work on larger runners (PR #117353)
Tom Stellard via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 22 13:52:22 PST 2024
https://github.com/tstellar updated https://github.com/llvm/llvm-project/pull/117353
>From eac6c1bcb47d8faf7a58b85915dc7c2c91df6f6c Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 22 Nov 2024 09:39:23 -0800
Subject: [PATCH 1/3] workflows: Rewrite build-ci-container to work on larger
runners
Also switch them over to the new depot runners.
---
.github/workflows/build-ci-container.yml | 52 ++-----------------
.../github-action-ci/stage1.Dockerfile | 25 ++++++++-
2 files changed, 27 insertions(+), 50 deletions(-)
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 28fc7de2ee0654..67e3a54f02b0ff 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -22,8 +22,10 @@ jobs:
# run this on the self-hosted runners and don't have to do it this way to
# avoid timeouts.
build-ci-container-stage1:
+ permissions:
+ packages: write
if: github.repository_owner == 'llvm'
- runs-on: ubuntu-latest
+ runs-on: depot-ubuntu-24.04-16
steps:
- name: Checkout LLVM
uses: actions/checkout at v4
@@ -40,22 +42,6 @@ jobs:
working-directory: ./.github/workflows/containers/github-action-ci/
run: |
podman build -t stage1-toolchain --target stage1-toolchain -f stage1.Dockerfile .
- - name: Save container image
- run: |
- podman save stage1-toolchain > stage1-toolchain.tar
- - name: Upload container image
- uses: actions/upload-artifact at v4
- with:
- name: stage1-toolchain
- path: stage1-toolchain.tar
- retention-days: 1
- build-ci-container-stage2:
- if: github.repository_owner == 'llvm'
- runs-on: ubuntu-latest
- needs: build-ci-container-stage1
- permissions:
- packages: write
- steps:
- name: Write Variables
id: vars
run: |
@@ -64,38 +50,6 @@ jobs:
echo "container-name=$container_name" >> $GITHUB_OUTPUT
echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
- - name: Checkout LLVM
- uses: actions/checkout at v4
- with:
- sparse-checkout: .github/workflows/containers/github-action-ci/
-
- - name: Change podman Root Direcotry
- run: |
- mkdir -p ~/.config/containers
- sudo mkdir -p /mnt/podman
- sudo chown `whoami`:`whoami` /mnt/podman
- cp ./.github/workflows/containers/github-action-ci/storage.conf ~/.config/containers/storage.conf
- podman info
-
- # Download the container image into /mnt/podman rather than
- # $GITHUB_WORKSPACE to avoid space limitations on the default drive
- # and use the permissions setup for /mnt/podman.
- - name: Download stage1-toolchain
- uses: actions/download-artifact at v4
- with:
- name: stage1-toolchain
- path: /mnt/podman
-
- - name: Load stage1-toolchain
- run: |
- podman load -i /mnt/podman/stage1-toolchain.tar
-
- - name: Build Container
- working-directory: ./.github/workflows/containers/github-action-ci/
- run: |
- podman build -t ${{ steps.vars.outputs.container-name-tag }} -f stage2.Dockerfile .
- podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest
-
- name: Test Container
run: |
for image in ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}; do
diff --git a/.github/workflows/containers/github-action-ci/stage1.Dockerfile b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
index 3e2c1ab11d58bf..132ef214ada5d8 100644
--- a/.github/workflows/containers/github-action-ci/stage1.Dockerfile
+++ b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
@@ -39,4 +39,27 @@ RUN cmake -B ./build -G Ninja ./llvm \
-DCLANG_DEFAULT_LINKER="lld" \
-DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=/llvm-project-llvmorg-$LLVM_VERSION/llvm
-RUN ninja -C ./build stage2-instrumented-clang stage2-instrumented-lld
+RUN ninja -C ./build stage2-instrumented-clang stage2-instrumented-lld stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution && rm -rf ./build
+
+FROM base
+
+COPY --from=stage1-toolchain $LLVM_SYSROOT $LLVM_SYSROOT
+
+# Need to install curl for hendrikmuhs/ccache-action
+# Need nodejs for some of the GitHub actions.
+# Need perl-modules for clang analyzer tests.
+# Need git for SPIRV-Tools tests.
+RUN apt-get update && \
+ apt-get install -y \
+ binutils \
+ cmake \
+ curl \
+ git \
+ libstdc++-11-dev \
+ ninja-build \
+ nodejs \
+ perl-modules \
+ python3-psutil
+
+ENV LLVM_SYSROOT=$LLVM_SYSROOT
+ENV PATH=${LLVM_SYSROOT}/bin:${PATH}
>From 62fac26ae201095ee771aa9c0208faccdc97e955 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 22 Nov 2024 10:02:08 -0800
Subject: [PATCH 2/3] Remove unneeded target
---
.github/workflows/containers/github-action-ci/stage1.Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/containers/github-action-ci/stage1.Dockerfile b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
index 132ef214ada5d8..b5403356bb612b 100644
--- a/.github/workflows/containers/github-action-ci/stage1.Dockerfile
+++ b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
@@ -39,7 +39,7 @@ RUN cmake -B ./build -G Ninja ./llvm \
-DCLANG_DEFAULT_LINKER="lld" \
-DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=/llvm-project-llvmorg-$LLVM_VERSION/llvm
-RUN ninja -C ./build stage2-instrumented-clang stage2-instrumented-lld stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution && rm -rf ./build
+RUN ninja -C ./build stage2-instrumented-clang stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution && rm -rf ./build
FROM base
>From 7f1cd6dadab2b40f6acd71f793689e15a7039253 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 22 Nov 2024 21:46:47 +0000
Subject: [PATCH 3/3] Fix targets and apply new patch
---
.../containers/github-action-ci/bootstrap.patch | 13 -------------
.../containers/github-action-ci/stage1.Dockerfile | 15 ++++++---------
2 files changed, 6 insertions(+), 22 deletions(-)
delete mode 100644 .github/workflows/containers/github-action-ci/bootstrap.patch
diff --git a/.github/workflows/containers/github-action-ci/bootstrap.patch b/.github/workflows/containers/github-action-ci/bootstrap.patch
deleted file mode 100644
index 55631c54a396f1..00000000000000
--- a/.github/workflows/containers/github-action-ci/bootstrap.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/clang/cmake/caches/BOLT-PGO.cmake b/clang/cmake/caches/BOLT-PGO.cmake
-index 1a04ca9a74e5..d092820e4115 100644
---- a/clang/cmake/caches/BOLT-PGO.cmake
-+++ b/clang/cmake/caches/BOLT-PGO.cmake
-@@ -4,6 +4,8 @@ set(CLANG_BOOTSTRAP_TARGETS
- stage2-clang-bolt
- stage2-distribution
- stage2-install-distribution
-+ clang
-+ lld
- CACHE STRING "")
- set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS
- clang-bolt
diff --git a/.github/workflows/containers/github-action-ci/stage1.Dockerfile b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
index b5403356bb612b..69dd04fc01ef57 100644
--- a/.github/workflows/containers/github-action-ci/stage1.Dockerfile
+++ b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
@@ -19,13 +19,11 @@ RUN curl -O -L https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-$L
WORKDIR /llvm-project-llvmorg-$LLVM_VERSION
-COPY bootstrap.patch /
+# Patch to enable better PGO profile data.
+# TODO: Remove this for llvm 20
+ADD https://github.com/llvm/llvm-project/commit/738250989ce516f02f809bdfde474a039c77e81f.patch .
-# TODO(boomanaiden154): Remove the bootstrap patch once we unsplit the build
-# and no longer need to explicitly build the stage2 dependencies.
-RUN cat /bootstrap.patch | patch -p1
-
-RUN mkdir build
+RUN patch -p1 < 738250989ce516f02f809bdfde474a039c77e81f.patch
RUN cmake -B ./build -G Ninja ./llvm \
-C ./clang/cmake/caches/BOLT-PGO.cmake \
@@ -36,10 +34,9 @@ RUN cmake -B ./build -G Ninja ./llvm \
-DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \
-DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \
-DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build" \
- -DCLANG_DEFAULT_LINKER="lld" \
- -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=/llvm-project-llvmorg-$LLVM_VERSION/llvm
+ -DCLANG_DEFAULT_LINKER="lld"
-RUN ninja -C ./build stage2-instrumented-clang stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution && rm -rf ./build
+RUN ninja -C ./build stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution && rm -rf ./build
FROM base
More information about the llvm-commits
mailing list