[llvm] workflows: Refactor release-tasks.yml (PR #69523)
Tom Stellard via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 30 13:05:43 PDT 2023
https://github.com/tstellar updated https://github.com/llvm/llvm-project/pull/69523
>From b5fe532633e8d0b4057c46aefa2366788ab7726e Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Wed, 18 Oct 2023 02:25:10 -0700
Subject: [PATCH 1/2] workflows/release-tasks: Fix release note artifact upload
---
.github/workflows/release-tasks.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml
index 065b84dd8822eb7..85b720e323d1b1d 100644
--- a/.github/workflows/release-tasks.yml
+++ b/.github/workflows/release-tasks.yml
@@ -48,7 +48,7 @@ jobs:
./llvm/utils/release/github-upload-release.py --token ${{ github.token }} --release ${{ steps.validate-tag.outputs.release-version }} upload --files ./*doxygen*.tar.xz
- name: Create Release Notes Artifact
- uses: actions/download-artifact at v3
+ uses: actions/upload-artifact at v3
with:
name: release-notes
path: docs-build/html-export/
>From 90b3368218fd547906056c16a0e30361cc13bbb7 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Wed, 18 Oct 2023 04:56:28 -0700
Subject: [PATCH 2/2] workflows: Refactor release-tasks.yml
* Split out the lit release job and the documentation build jobs into their
own workflow files. This makes it possible to manually run these jobs
via workflow_dispatch.
* Improve tag/user validation and ensure it gets run for each release
task.
---
.github/workflows/release-binaries.yml | 51 ++++---
.github/workflows/release-documentation.yml | 84 +++++++++++
.github/workflows/release-doxygen.yml | 60 ++++++++
.github/workflows/release-lit.yml | 74 +++++++++
.github/workflows/release-tasks.yml | 140 ++++++------------
.../workflows/set-release-binary-outputs.sh | 18 +--
llvm/utils/release/github-upload-release.py | 21 ++-
7 files changed, 318 insertions(+), 130 deletions(-)
create mode 100644 .github/workflows/release-documentation.yml
create mode 100644 .github/workflows/release-doxygen.yml
create mode 100644 .github/workflows/release-lit.yml
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index e52e52f5d3f36fa..e9155e355fd0ccb 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -1,20 +1,29 @@
name: Release Binaries
on:
- push:
- tags:
- - 'llvmorg-*'
workflow_dispatch:
inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
upload:
description: 'Upload binaries to the release page'
required: true
- default: true
+ default: false
type: boolean
- tag:
- description: 'Tag to build'
+
+ workflow_call:
+ inputs:
+ release-version:
+ description: 'Release Version'
required: true
type: string
+ upload:
+ description: 'Upload binaries to the release page'
+ required: true
+ default: false
+ type: boolean
permissions:
contents: read # Default everything to read-only
@@ -23,21 +32,26 @@ jobs:
prepare:
name: Prepare to build binaries
runs-on: ubuntu-22.04
- if: github.repository == 'llvm/llvm-project'
outputs:
- release-version: ${{ steps.validate-tag.outputs.release-version }}
- release: ${{ steps.validate-tag.outputs.release }}
- build-dir: ${{ steps.validate-tag.outputs.build-dir }}
- rc-flags: ${{ steps.validate-tag.outputs.rc-flags }}
- ref: ${{ steps.validate-tag.outputs.ref }}
- upload: ${{ steps.validate-tag.outputs.upload }}
+ release-version: ${{ steps.vars.outputs.release-version }}
+ release: ${{ steps.vars.outputs.release }}
+ build-dir: ${{ steps.vars.outputs.build-dir }}
+ rc-flags: ${{ steps.vars.outputs.rc-flags }}
+ ref: ${{ steps.vars.outputs.ref }}
+ upload: ${{ steps.vars.outputs.upload }}
steps:
- name: Checkout LLVM
uses: actions/checkout at v4
- - name: Validate and parse tag
- id: validate-tag
+ - name: Check Permissions
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} check-permissions
+
+ - name: Collect Variables
+ id: vars
# In order for the test-release.sh script to run correctly, the LLVM
# source needs to be at the following location relative to the build dir:
# | X.Y.Z-rcN | ./rcN/llvm-project
@@ -47,15 +61,12 @@ jobs:
# | X.Y.Z-rcN | -rc N -test-asserts
# | X.Y.Z | -final
run: |
- tag="${{ github.ref_name }}"
- trimmed=$(echo ${{ inputs.tag }} | xargs)
- [[ "$trimmed" != "" ]] && tag="$trimmed"
if [ -n "${{ inputs.upload }}" ]; then
upload="${{ inputs.upload }}"
else
- upload="true"
+ upload="false"
fi
- bash .github/workflows/set-release-binary-outputs.sh "${{ github.actor }}" "$tag" "$upload"
+ bash .github/workflows/set-release-binary-outputs.sh "${{ inputs.release-version }}" "$upload"
build-binaries:
name: ${{ matrix.target.triple }}
diff --git a/.github/workflows/release-documentation.yml b/.github/workflows/release-documentation.yml
new file mode 100644
index 000000000000000..832570ed6fe93f6
--- /dev/null
+++ b/.github/workflows/release-documentation.yml
@@ -0,0 +1,84 @@
+name: Release Documentation
+
+permissions:
+ contents: read
+
+on:
+ workflow_dispatch:
+ inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
+ upload:
+ description: 'Upload documentation'
+ required: false
+ type: boolean
+
+ workflow_call:
+ inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
+ upload:
+ description: 'Upload documentation'
+ required: false
+ type: boolean
+
+jobs:
+ release-documentation:
+ name: Build and Upload Release Documentation
+ runs-on: ubuntu-latest
+ env:
+ upload: ${{ inputs.upload && !contains(inputs.release-version, 'rc') }}
+ steps:
+ - name: Checkout LLVM
+ uses: actions/checkout at v4
+
+ - name: Install Dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y \
+ doxygen \
+ graphviz \
+ python3-github \
+ ninja-build \
+ texlive-font-utils
+ pip3 install --user -r ./llvm/docs/requirements.txt
+ echo "UPLOAD SETTING: $upload"
+
+ - name: Build Documentation
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-doxygen
+
+ - name: Create Release Notes Artifact
+ uses: actions/upload-artifact at v3
+ with:
+ name: release-notes
+ path: docs-build/html-export/
+
+ - name: Clone www-releases
+ if: env.upload
+ uses: actions/checkout at v4
+ with:
+ repository: ${{ github.repository_owner }}/www-releases
+ ref: main
+ fetch-depth: 0
+ path: www-releases
+
+ - name: Upload Release Notes
+ if: env.upload
+ env:
+ WWW_RELEASES_TOKEN: ${{ secrets.WWW_RELEASES_TOKEN }}
+ run: |
+ mkdir -p ../www-releases/${{ inputs.release-version }}
+ mv ./docs-build/html-export/* ../www-releases/${{ inputs.release-version }}
+ cd ../www-releases
+ git add ${{ inputs.release-version }}
+ git config user.email "llvmbot at llvm.org"
+ git config user.name "llvmbot"
+ git commit -a -m "Add ${{ inputs.release-version }} documentation"
+ git push "https://$WWW_RELEASES_TOKEN@github.com/${{ github.repository_owner }}/www-releases" main:main
diff --git a/.github/workflows/release-doxygen.yml b/.github/workflows/release-doxygen.yml
new file mode 100644
index 000000000000000..62907f3f5c6937c
--- /dev/null
+++ b/.github/workflows/release-doxygen.yml
@@ -0,0 +1,60 @@
+name: Release Doxygen
+
+permissions:
+ contents: write
+
+on:
+ workflow_dispatch:
+ inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
+ upload:
+ description: 'Upload documentation'
+ required: false
+ type: boolean
+
+ workflow_call:
+ inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
+ upload:
+ description: 'Upload documentation'
+ required: false
+ type: boolean
+
+jobs:
+ release-doxygen:
+ name: Build and Upload Release Doxygen
+ runs-on: ubuntu-latest
+ env:
+ upload: ${{ inputs.upload && !contains(inputs.release-version, 'rc') }}
+ steps:
+ - name: Checkout LLVM
+ uses: actions/checkout at v4
+
+ - name: Install Dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y \
+ doxygen \
+ graphviz \
+ python3-github \
+ ninja-build \
+ texlive-font-utils
+ pip3 install --user -r ./llvm/docs/requirements.txt
+
+ - name: Build Doxygen
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ echo "UPLOAD SETTING: $upload"
+ ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-sphinx
+
+ - name: Upload Doxygen
+ if: env.upload
+ run: |
+ ./llvm/utils/release/github-upload-release.py --token "$GITHUB_TOKEN" --release "${{ inputs.release-version }}" --user "${{ github.actor }}" upload --files ./*doxygen*.tar.xz
diff --git a/.github/workflows/release-lit.yml b/.github/workflows/release-lit.yml
new file mode 100644
index 000000000000000..feaca8719608640
--- /dev/null
+++ b/.github/workflows/release-lit.yml
@@ -0,0 +1,74 @@
+name: Release Lit
+
+permissions:
+ contents: read
+
+on:
+ workflow_dispatch:
+ inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
+
+ workflow_call:
+ inputs:
+ release-version:
+ description: 'Release Version'
+ required: true
+ type: string
+
+jobs:
+ release-lit:
+ name: Release Lit
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout LLVM
+ uses: actions/checkout at v4
+ with:
+ ref: "llvmorg-${{ inputs.release-version }}"
+
+ - name: Install dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y python3-setuptools python3-psutil python3-github
+
+ - name: Check Permissions
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} check-permissions
+
+ - name: Setup Cpp
+ uses: aminya/setup-cpp at v1
+ with:
+ compiler: llvm-16.0.6
+ cmake: true
+ ninja: true
+
+ - name: Test lit
+ run: |
+ mkdir build && cd build
+ export FILECHECK_OPTS='-dump-input-filter=all -vv -color'
+ cmake ../llvm -DCMAKE_BUILD_TYPE=Release -G Ninja
+ ninja -v -j $(nproc) check-lit
+
+ - name: Package lit
+ run: |
+ cd llvm/utils/lit
+ # Remove 'dev' suffix from lit version.
+ sed -i 's/ + "dev"//g' lit/__init__.py
+ python3 setup.py sdist
+
+ - name: Upload lit to test.pypi.org
+ uses: pypa/gh-action-pypi-publish at release/v1
+ with:
+ password: ${{ secrets.LLVM_LIT_TEST_PYPI_API_TOKEN }}
+ repository-url: https://test.pypi.org/legacy/
+ packages-dir: llvm/utils/lit/dist/
+
+ - name: Upload lit to pypi.org
+ uses: pypa/gh-action-pypi-publish at release/v1
+ with:
+ password: ${{ secrets.LLVM_LIT_PYPI_API_TOKEN }}
+ packages-dir: llvm/utils/lit/dist/
diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml
index 85b720e323d1b1d..4bc883623d4b1bf 100644
--- a/.github/workflows/release-tasks.yml
+++ b/.github/workflows/release-tasks.yml
@@ -1,7 +1,7 @@
name: Release Task
permissions:
- contents: read
+ contents: write
on:
push:
@@ -10,112 +10,70 @@ on:
- 'llvmorg-*'
jobs:
- release-tasks:
- permissions:
- contents: write # To upload assets to release.
+ validate-tag:
+ name: Validate Tag
runs-on: ubuntu-latest
if: github.repository == 'llvm/llvm-project'
+ outputs:
+ release-version: ${{ steps.validate-tag.outputs.release-version }}
steps:
- name: Validate Tag
id: validate-tag
run: |
- test "${{ github.actor }}" = "tstellar" || test "${{ github.actor }}" = "tru"
echo "${{ github.ref_name }}" | grep -e '^llvmorg-[0-9]\+\.[0-9]\+\.[0-9]\+\(-rc[0-9]\+\)\?$'
release_version=$(echo "${{ github.ref_name }}" | sed 's/llvmorg-//g')
echo "release-version=$release_version" >> "$GITHUB_OUTPUT"
- - name: Checkout LLVM
- uses: actions/checkout at v4
-
+ release-create:
+ name: Create a New Release
+ runs-on: ubuntu-latest
+ needs: validate-tag
+ steps:
- name: Install Dependencies
run: |
sudo apt-get update
- sudo apt-get install -y \
- doxygen \
- graphviz \
- python3-github \
- ninja-build \
- texlive-font-utils
- pip3 install --user -r ./llvm/docs/requirements.txt
-
- - name: Create Release
- run: |
- ./llvm/utils/release/./github-upload-release.py --token ${{ github.token }} --release ${{ steps.validate-tag.outputs.release-version }} create
-
- - name: Build Documentation
- run: |
- ./llvm/utils/release/build-docs.sh -release ${{ steps.validate-tag.outputs.release-version }}
- ./llvm/utils/release/github-upload-release.py --token ${{ github.token }} --release ${{ steps.validate-tag.outputs.release-version }} upload --files ./*doxygen*.tar.xz
-
- - name: Create Release Notes Artifact
- uses: actions/upload-artifact at v3
- with:
- name: release-notes
- path: docs-build/html-export/
-
- - name: Clone www-releases
- if: ${{ !contains(steps.validate-tag.outputs.release-version, 'rc') }}
- uses: actions/checkout at v4
- with:
- repository: ${{ github.repository_owner }}/www-releases
- ref: main
- fetch-depth: 0
- path: www-releases
-
- - name: Upload Release Notes
- if: ${{ !contains(steps.validate-tag.outputs.release-version, 'rc') }}
- run: |
- mkdir -p ../www-releases/${{ steps.validate-tag.outputs.release-version }}
- mv ./docs-build/html-export/* ../www-releases/${{ steps.validate-tag.outputs.release-version }}
- cd ../www-releases
- git add ${{ steps.validate-tag.outputs.release-version }}
- git config user.email "llvmbot at llvm.org"
- git config user.name "llvmbot"
- git commit -a -m "Add ${{ steps.validate-tag.outputs.release-version }} documentation"
- git push https://${{ secrets.WWW_RELEASES_TOKEN }}@github.com/${{ github.repository_owner }}/www-releases main:main
+ sudo apt-get install python3-github
- release-lit:
- runs-on: ubuntu-latest
- if: github.repository == 'llvm/llvm-project'
- steps:
- name: Checkout LLVM
uses: actions/checkout at v4
- - name: Setup Cpp
- uses: aminya/setup-cpp at v1
- with:
- compiler: llvm-16.0.6
- cmake: true
- ninja: true
-
- - name: Install dependencies
- run: |
- sudo apt-get update
- sudo apt-get install -y python3-setuptools python3-psutil
-
- - name: Test lit
- run: |
- mkdir build && cd build
- export FILECHECK_OPTS='-dump-input-filter=all -vv -color'
- cmake ../llvm -DCMAKE_BUILD_TYPE=Release -G Ninja
- ninja -v -j $(nproc) check-lit
-
- - name: Package lit
+ - name: Create Release
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
run: |
- cd llvm/utils/lit
- # Remove 'dev' suffix from lit version.
- sed -i 's/ + "dev"//g' lit/__init__.py
- python3 setup.py sdist
-
- - name: Upload lit to test.pypi.org
- uses: pypa/gh-action-pypi-publish at release/v1
- with:
- password: ${{ secrets.LLVM_LIT_TEST_PYPI_API_TOKEN }}
- repository-url: https://test.pypi.org/legacy/
- packages-dir: llvm/utils/lit/dist/
+ ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --release ${{ needs.validate-tag.outputs.release-version }} --user ${{ github.actor }} create
+ release-documentation:
+ name: Build and Upload Release Documentation
+ needs:
+ - validate-tag
+ uses: ./.github/workflows/release-documentation.yml
+ with:
+ release-version: ${{ needs.validate-tag.outputs.release-version }}
+ upload: true
+
+ release-doxygen:
+ name: Build and Upload Release Doxygen
+ needs:
+ - validate-tag
+ - release-create
+ uses: ./.github/workflows/release-doxygen.yml
+ with:
+ release-version: ${{ needs.validate-tag.outputs.release-version }}
+ upload: true
- - name: Upload lit to pypi.org
- uses: pypa/gh-action-pypi-publish at release/v1
- with:
- password: ${{ secrets.LLVM_LIT_PYPI_API_TOKEN }}
- packages-dir: llvm/utils/lit/dist/
+ release-lit:
+ name: Release Lit
+ needs: validate-tag
+ uses: ./.github/workflows/release-lit.yml
+ with:
+ release-version: ${{ needs.validate-tag.outputs.release-version }}
+
+ release-binaries:
+ name: Build Release Binaries
+ needs:
+ - validate-tag
+ - release-create
+ uses: ./.github/workflows/release-binaries.yml
+ with:
+ release-version: ${{ needs.validate-tag.outputs.release-version }}
+ upload: true
diff --git a/.github/workflows/set-release-binary-outputs.sh b/.github/workflows/set-release-binary-outputs.sh
index 8a7944e7e55fa06..0fa23fe3df968c1 100644
--- a/.github/workflows/set-release-binary-outputs.sh
+++ b/.github/workflows/set-release-binary-outputs.sh
@@ -8,21 +8,9 @@ if [ -z "$GITHUB_OUTPUT" ]; then
echo "Writing output variables to $GITHUB_OUTPUT"
fi
-github_user=$1
-tag=$2
-upload=$3
-
-if [[ "$github_user" != "tstellar" && "$github_user" != "tru" ]]; then
- echo "ERROR: User not allowed: $github_user"
- exit 1
-fi
-pattern='^llvmorg-[0-9]\+\.[0-9]\+\.[0-9]\+\(-rc[0-9]\+\)\?$'
-echo "$tag" | grep -e $pattern
-if [ $? != 0 ]; then
- echo "ERROR: Tag '$tag' doesn't match pattern: $pattern"
- exit 1
-fi
-release_version=`echo "$tag" | sed 's/llvmorg-//g'`
+release_version=$1
+upload=$2
+tag="llvmorg-$release_version"
release=`echo "$release_version" | sed 's/-.*//g'`
build_dir=`echo "$release_version" | sed 's,^[^-]\+,final,' | sed 's,[^-]\+-rc\(.\+\),rc\1,'`
rc_flags=`echo "$release_version" | sed 's,^[^-]\+,-final,' | sed 's,[^-]\+-rc\(.\+\),-rc \1 -test-asserts,' | sed 's,--,-,'`
diff --git a/llvm/utils/release/github-upload-release.py b/llvm/utils/release/github-upload-release.py
index 86a71368dd84322..95d852e08c9cc34 100755
--- a/llvm/utils/release/github-upload-release.py
+++ b/llvm/utils/release/github-upload-release.py
@@ -30,6 +30,7 @@
import argparse
import github
+import sys
def create_release(repo, release, tag=None, name=None, message=None):
@@ -56,22 +57,34 @@ def upload_files(repo, release, files):
parser = argparse.ArgumentParser()
-parser.add_argument("command", type=str, choices=["create", "upload"])
+parser.add_argument("command", type=str, choices=["create", "upload", "check-permissions"])
# All args
parser.add_argument("--token", type=str)
parser.add_argument("--release", type=str)
+parser.add_argument("--user", type=str)
# Upload args
parser.add_argument("--files", nargs="+", type=str)
-
args = parser.parse_args()
github = github.Github(args.token)
-llvm_repo = github.get_organization("llvm").get_repo("llvm-project")
+llvm_org = github.get_organization("llvm")
+llvm_repo = llvm_org.get_repo("llvm-project")
+
+if args.user:
+ # Validate that this user is allowed to modify releases.
+ user = github.get_user(args.user)
+ team = llvm_org.get_team_by_slug('llvm-release-managers')
+ if not team.has_in_members(user):
+ print("User {} is not a allowed to modify releases".format(args.user))
+ sys.exit(1)
+elif args.command == "check-permissions":
+ print("--user option required for check-permissions")
+ sys.exit(1)
if args.command == "create":
- create_release(llvm_repo, args.release)
+ create_release(llvm_repo, args.release, args.user)
if args.command == "upload":
upload_files(llvm_repo, args.release, args.files)
More information about the llvm-commits
mailing list