[llvm] [GitHub][CI] Factor out duplicate container building code into composite actions (PR #166663)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 5 19:46:38 PST 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-github-workflow
Author: Tom Stellard (tstellar)
<details>
<summary>Changes</summary>
---
Full diff: https://github.com/llvm/llvm-project/pull/166663.diff
4 Files Affected:
- (modified) .github/workflows/build-ci-container-tooling.yml (+27-69)
- (modified) .github/workflows/build-ci-container.yml (+30-76)
- (added) .github/workflows/build-container/action.yml (+95)
- (added) .github/workflows/push-container/action.yml (+44)
``````````diff
diff --git a/.github/workflows/build-ci-container-tooling.yml b/.github/workflows/build-ci-container-tooling.yml
index 992947eb2fffb..040cae52854fa 100644
--- a/.github/workflows/build-ci-container-tooling.yml
+++ b/.github/workflows/build-ci-container-tooling.yml
@@ -12,17 +12,29 @@ on:
- '.github/workflows/containers/github-action-ci-tooling/**'
- llvm/utils/git/requirements_formatting.txt
- llvm/utils/git/requirements_linting.txt
+ - '.github/workflows/build-container/**'
+ - '.github/workflows/push-container/**'
pull_request:
paths:
- .github/workflows/build-ci-container-tooling.yml
- '.github/workflows/containers/github-action-ci-tooling/**'
- llvm/utils/git/requirements_formatting.txt
- llvm/utils/git/requirements_linting.txt
+ - '.github/workflows/build-container/**'
+ - '.github/workflows/push-container/**'
jobs:
build-ci-container-tooling:
if: github.repository_owner == 'llvm'
runs-on: ubuntu-24.04
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - container-name: code-format
+ test-command: 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black'
+ - container-name: code-lint
+ test-command: 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage'
steps:
- name: Checkout LLVM
uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -32,48 +44,15 @@ jobs:
llvm/utils/git/requirements_formatting.txt
llvm/utils/git/requirements_linting.txt
clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
+ .github/workflows/build-container
- - name: Write Variables
- id: vars
- run: |
- tag=$(git rev-parse --short=12 HEAD)
- container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04"
- echo "container-name-format=$container_name-code-format" >> $GITHUB_OUTPUT
- echo "container-name-lint=$container_name-code-lint" >> $GITHUB_OUTPUT
- echo "container-name-format-tag=$container_name-format:$tag" >> $GITHUB_OUTPUT
- echo "container-name-lint-tag=$container_name-lint:$tag" >> $GITHUB_OUTPUT
- echo "container-format-filename=$(echo $container_name-format:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
- echo "container-lint-filename=$(echo $container_name-lint:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
-
- - name: Build container
- run: |
- podman build --target ci-container-code-format \
- -f .github/workflows/containers/github-action-ci-tooling/Dockerfile \
- -t ${{ steps.vars.outputs.container-name-format-tag }} .
- podman build --target ci-container-code-lint \
- -f .github/workflows/containers/github-action-ci-tooling/Dockerfile \
- -t ${{ steps.vars.outputs.container-name-lint-tag }} .
-
- # Save the container so we have it in case the push fails. This also
- # allows us to separate the push step into a different job so we can
- # maintain minimal permissions while building the container.
- - name: Save container image
- run: |
- podman save ${{ steps.vars.outputs.container-name-format-tag }} > ${{ steps.vars.outputs.container-format-filename }}
- podman save ${{ steps.vars.outputs.container-name-lint-tag }} > ${{ steps.vars.outputs.container-lint-filename }}
-
- - name: Upload container image
- uses: actions/upload-artifact at ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ - name: Build Container
+ uses: ./.github/workflows/build-container
with:
- name: container-amd64
- path: "*.tar"
- retention-days: 14
-
- - name: Test Container
- run: |
- # Use --pull=never to ensure we are testing the just built image.
- podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-format-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black'
- podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-lint-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage'
+ container-name: ci-ubuntu-24.04-${{ matrix.container-name }}
+ dockerfile: .github/workflows/containers/github-action-ci-tooling/Dockerfile
+ target: ci-container-${{ matrix.container-name }}
+ test-command: ${{ matrix.test-command }}
push-ci-container:
if: github.event_name == 'push'
@@ -82,34 +61,13 @@ jobs:
permissions:
packages: write
runs-on: ubuntu-24.04
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- - name: Download container
- uses: actions/download-artifact at 634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
-
- - name: Push Container
- run: |
- function push_container {
- image_name=$1
- latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
- podman tag $image_name $latest_name
- echo "Pushing $image_name ..."
- podman push $image_name
- echo "Pushing $latest_name ..."
- podman push $latest_name
- }
-
- podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
- for f in $(find . -iname '*.tar'); do
- image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
- push_container $image_name
+ - name: Checkout LLVM
+ uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ with:
+ sparse-checkout: |
+ .github/workflows/push-container
- if echo $image_name | grep '/amd64/'; then
- # For amd64, create an alias with the arch component removed.
- # This matches the convention used on dockerhub.
- default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
- podman tag $image_name $default_image_name
- push_container $default_image_name
- fi
- done
+ - uses: ./.github/workflows/push-container
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 027c558afdd0b..5ef28dcac8b21 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -10,10 +10,14 @@ on:
paths:
- .github/workflows/build-ci-container.yml
- '.github/workflows/containers/github-action-ci/**'
+ - '.github/workflows/build-container/**'
+ - '.github/workflows/push-container/**'
pull_request:
paths:
- .github/workflows/build-ci-container.yml
- '.github/workflows/containers/github-action-ci/**'
+ - '.github/workflows/build-container/**'
+ - '.github/workflows/push-container/**'
jobs:
build-ci-container:
@@ -21,61 +25,30 @@ jobs:
runs-on: ${{ matrix.runs-on }}
strategy:
matrix:
- include:
- # The arch names should match the names used on dockerhub.
- # See https://github.com/docker-library/official-images#architectures-other-than-amd64
- - arch: amd64
- runs-on: depot-ubuntu-24.04-16
- - arch: arm64v8
- runs-on: depot-ubuntu-24.04-arm-16
+ runs-on:
+ - depot-ubuntu-24.04-16
+ - depot-ubuntu-24.04-arm-16
+ container-name:
+ - ''
+ - agent
+ test-command:
+ - cd $HOME && printf '#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }' | clang++ -x c++ - && ./a.out | grep Hello
steps:
- name: Checkout LLVM
uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
- sparse-checkout: .github/workflows/containers/github-action-ci/
- # podman is not installed by default on the ARM64 images.
- - name: Install Podman
- if: runner.arch == 'ARM64'
- run: |
- sudo apt-get install podman
- - name: Write Variables
- id: vars
- run: |
- tag=$(git rev-parse --short=12 HEAD)
- container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/${{ matrix.arch }}/ci-ubuntu-24.04"
- echo "container-name=$container_name" >> $GITHUB_OUTPUT
- echo "container-name-agent=$container_name-agent" >> $GITHUB_OUTPUT
- echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
- echo "container-name-agent-tag=$container_name-agent:$tag" >> $GITHUB_OUTPUT
- echo "container-filename=$(echo $container_name:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
- echo "container-agent-filename=$(echo $container_name-agent:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
- - name: Build container
- working-directory: ./.github/workflows/containers/github-action-ci/
- run: |
- podman build --target ci-container -t ${{ steps.vars.outputs.container-name-tag }} .
- podman build --target ci-container-agent -t ${{ steps.vars.outputs.container-name-agent-tag }} .
+ sparse-checkout: |
+ .github/workflows/containers/github-action-ci/
+ .github/workflows/build-container
- # Save the container so we have it in case the push fails. This also
- # allows us to separate the push step into a different job so we can
- # maintain minimal permissions while building the container.
- - name: Save container image
- run: |
- podman save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }}
- podman save ${{ steps.vars.outputs.container-name-agent-tag }} > ${{ steps.vars.outputs.container-agent-filename }}
-
- - name: Upload container image
- uses: actions/upload-artifact at ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ - name: Build Container
+ uses: ./.github/workflows/build-container
with:
- name: container-${{ matrix.arch }}
- path: "*.tar"
- retention-days: 14
-
- - name: Test Container
- run: |
- for image in ${{ steps.vars.outputs.container-name-tag }}; do
- # Use --pull=never to ensure we are testing the just built image.
- podman run --pull=never --rm -it $image /usr/bin/bash -x -c 'cd $HOME && printf '\''#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello'
- done
+ container-name: ci-ubuntu-24.04${{ matrix.container-name && format('-{0}', matrix.container-name)}}
+ context: .github/workflows/containers/github-action-ci/
+ dockerfile: .github/workflows/containers/github-action-ci/Dockerfile
+ target: ci-container${{ matrix.container-name && format('-{0}', matrix.container-name) }}
+ test-command: ${{ matrix.test-command }}
push-ci-container:
if: github.event_name == 'push'
@@ -87,31 +60,12 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- - name: Download container
- uses: actions/download-artifact at 634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
-
- - name: Push Container
- run: |
- function push_container {
- image_name=$1
- latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
- podman tag $image_name $latest_name
- echo "Pushing $image_name ..."
- podman push $image_name
- echo "Pushing $latest_name ..."
- podman push $latest_name
- }
-
- podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
- for f in $(find . -iname '*.tar'); do
- image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
- push_container $image_name
+ - name: Checkout LLVM
+ uses: actions/checkout at 08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+ with:
+ sparse-checkout: |
+ .github/workflows/push-container
- if echo $image_name | grep '/amd64/'; then
- # For amd64, create an alias with the arch component removed.
- # This matches the convention used on dockerhub.
- default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
- podman tag $image_name $default_image_name
- push_container $default_image_name
- fi
- done
+ - uses: ./.github/workflows/push-container
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/build-container/action.yml b/.github/workflows/build-container/action.yml
new file mode 100644
index 0000000000000..c25fc41589047
--- /dev/null
+++ b/.github/workflows/build-container/action.yml
@@ -0,0 +1,95 @@
+name: Build Container
+description: >-
+ Build and test a container using the standard llvm naming scheme for containers.
+
+inputs:
+ tag:
+ description: >-
+ The tag to use for this container.
+ required: false
+ container-name:
+ description: >-
+ The name for the container.
+ required: true
+ dockerfile:
+ description: >-
+ Path to docker file.
+ required: false
+ target:
+ description: >-
+ The container target to build 'passed to podman via ---target option'
+ required: false
+ context:
+ description: >-
+ Path to context for the container build.
+ required: false
+ test-command:
+ description: >-
+ Test command to run to ensure the container is working correctly.
+ required: false
+
+runs:
+ using: "composite"
+ steps:
+ # podman is not installed by default on the ARM64 images.
+ - name: Install Podman
+ if: runner.arch == 'ARM64'
+ shell: bash
+ run: |
+ sudo apt-get install podman
+
+ - name: Build Container
+ shell: bash
+ env:
+ INPUT_TAG: ${{inputs.tag }}
+ INPUT_CONTAINER_NAME: ${{ inputs.container-name }}
+ INPUT_TARGET: ${{ inputs.target }}
+ INPUT_DOCKERFILE: ${{ inputs.dockerfile }}
+ INPUT_CONTEXT: ${{ inputs.context }}
+ id: build
+ run: |
+ env
+ tag="${INPUT_TAG:-$(git rev-parse --short=12 HEAD)}"
+
+ case "$RUNNER_ARCH" in
+ ARM64)
+ container_arch="arm64v8"
+ ;;
+ *)
+ container_arch="amd64"
+ ;;
+ esac
+
+ container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/$container_arch/$INPUT_CONTAINER_NAME:$tag"
+ container_filename="$(echo $container_name | sed -e 's/\//-/g' -e 's/:/-/g').tar"
+ if [ -n "$INPUT_TARGET" ]; then
+ podman_options="$podman_options --target $INPUT_TARGET"
+ fi
+ if [ -n "$INPUT_DOCKERFILE" ]; then
+ podman_options="$podman_options -f $INPUT_DOCKERFILE"
+ fi
+ podman_options="$podman_options ${INPUT_CONTEXT:-.}"
+ echo "Podman Options: $podman_options"
+
+ podman build -t $container_name $podman_options
+
+ podman save $container_name > $container_filename
+
+ echo "container-full-name=$container_name" >> $GITHUB_OUTPUT
+
+ - name: Create container artifact
+ uses: actions/upload-artifact at ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: ${{ inputs.container-name }}-${{ runner.arch }}
+ path: "*.tar"
+ retention-days: 14
+
+ - name: Test container
+ shell: bash
+ if: inputs.test-command
+ env:
+ INPUT_TEST_COMMAND: ${{ inputs.test-command }}
+ CONTAINER_FULL_NAME: ${{ steps.build.outputs.container-full-name }}
+ run: |
+ podman run --pull=never --rm -it $CONTAINER_FULL_NAME /usr/bin/bash -x -c "$INPUT_TEST_COMMAND"
+
diff --git a/.github/workflows/push-container/action.yml b/.github/workflows/push-container/action.yml
new file mode 100644
index 0000000000000..9f02d1ea205f1
--- /dev/null
+++ b/.github/workflows/push-container/action.yml
@@ -0,0 +1,44 @@
+name: Push Container
+description: >-
+ Download all container artifacts for this job and push them to the GitHub registry.
+
+inputs:
+ token:
+ description: >-
+ Token to use to authenticate with the container registry.
+ required: true
+
+runs:
+ using: "composite"
+ steps:
+ - name: Download container
+ uses: actions/download-artifact at 634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+
+ - name: Push Container
+ env:
+ GITHUB_TOKEN: ${{ inputs.token }}
+ shell: bash
+ run: |
+ function push_container {
+ image_name=$1
+ latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
+ podman tag $image_name $latest_name
+ echo "Pushing $image_name ..."
+ podman push $image_name
+ echo "Pushing $latest_name ..."
+ podman push $latest_name
+ }
+
+ podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
+ for f in $(find . -iname '*.tar'); do
+ image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
+ push_container $image_name
+
+ if echo $image_name | grep '/amd64/'; then
+ # For amd64, create an alias with the arch component removed.
+ # This matches the convention used on dockerhub.
+ default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
+ podman tag $image_name $default_image_name
+ push_container $default_image_name
+ fi
+ done
``````````
</details>
https://github.com/llvm/llvm-project/pull/166663
More information about the llvm-commits
mailing list