[llvm] [Github] Add a windows CI container (PR #118206)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 11 01:22:38 PST 2024
https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/118206
>From 1b9390a097b423ff6bbf3c2916efc232fe3fe7af Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Sun, 1 Dec 2024 10:20:39 +0000
Subject: [PATCH 1/6] [Github] Add a windows CI container
This patch adds a windows CI container mostly based off of the existing
container used for Buildkite (https://github.com/google/llvm-premerge-checks/blob/a687e33c37fbdcf67b52805c8cf3a8ed145e3243/containers/buildkite-windows/Dockerfile#L1).
This is intended to be a starting point as we transition to Github Actions
with the eventual plan being to build a custom windows toolchain similar
to what we do on Linux.
---
.../workflows/build-ci-container-windows.yml | 32 +++++
.../github-action-ci-windows/Dockerfile | 118 ++++++++++++++++++
2 files changed, 150 insertions(+)
create mode 100644 .github/workflows/build-ci-container-windows.yml
create mode 100644 .github/workflows/containers/github-action-ci-windows/Dockerfile
diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
new file mode 100644
index 00000000000000..013bc559864fe6
--- /dev/null
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -0,0 +1,32 @@
+name: Build Windows CI Container
+
+permissions:
+ contents: read
+
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - .github/workflows/build-ci-container-windows.yml
+ - '.github/workflows/containers/github-action-ci-windows/**'
+ pull_request:
+ branches:
+ - main
+ paths:
+ - .github/workflows/build-ci-container-windows.yml
+ - '.github/workflows/containers/github-action-ci-windows/**'
+
+jobs:
+ build-ci-container-windows:
+ if: github.repository_owner == 'llvm'
+ runs-on: windows-2019
+ steps:
+ - name: Checkout LLVM
+ uses: actions/checkout at v4
+ with:
+ sparse-checkout: .github/workflows/containers/github-action-ci-windows
+ - name: Build Container
+ working-directory: ./.github/workflows/container/github-action-ci-windows
+ run: |
+ docker build -t ci-windows-2019 .
diff --git a/.github/workflows/containers/github-action-ci-windows/Dockerfile b/.github/workflows/containers/github-action-ci-windows/Dockerfile
new file mode 100644
index 00000000000000..bc56e20935500b
--- /dev/null
+++ b/.github/workflows/containers/github-action-ci-windows/Dockerfile
@@ -0,0 +1,118 @@
+# Agent image for LLVM org cluster.
+# .net 4.8 is required by chocolately package manager.
+FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019
+
+# Restore the default Windows shell for correct batch processing.
+SHELL ["cmd", "/S", "/C"]
+
+# Download the Build Tools bootstrapper.
+ADD https://aka.ms/vs/16/release/vs_buildtools.exe /TEMP/vs_buildtools.exe
+
+RUN powershell -Command Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+
+# Download channel for fixed install.
+ARG CHANNEL_URL=https://aka.ms/vs/16/release/channel
+ADD ${CHANNEL_URL} /TEMP/VisualStudio.chman
+
+# Install Build Tools with C++ workload.
+# - Documentation for docker installation
+# https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019
+# - Documentation on workloads
+# https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools
+# - Documentation on flags
+# https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019
+RUN /TEMP/vs_buildtools.exe --quiet --wait --norestart --nocache \
+ --channelUri C:\TEMP\VisualStudio.chman \
+ --installChannelUri C:\TEMP\VisualStudio.chman \
+ --installPath C:\BuildTools \
+ --add Microsoft.VisualStudio.Workload.VCTools \
+ --add Microsoft.VisualStudio.Component.VC.ATL \
+ --includeRecommended \
+ || IF "%ERRORLEVEL%"=="3010" EXIT 0
+
+# Register DIA dll (Debug Interface Access) so it can be used to symbolize
+# the stack traces. Register dll for 32 and 64 bit.
+# see https://developercommunity.visualstudio.com/content/problem/290674/msdia140dll-is-not-registered-on-vs2017-hosts.html
+
+RUN regsvr32 /S "C:\BuildTools\DIA SDK\bin\amd64\msdia140.dll" & \
+ regsvr32 /S "C:\BuildTools\DIA SDK\bin\msdia140.dll"
+
+# install tools as described in https://llvm.org/docs/GettingStartedVS.html
+# and a few more that were not documented...
+RUN choco install -y ninja git
+# Pin an older version of Python; the current Python 3.10 fails when
+# doing "pip install" for the other dependencies, as it fails to find libxml
+# while compiling some package.
+RUN choco install -y python3 --version 3.9.7
+
+# ActivePerl is currently not installable via Chocolatey, see
+# http://disq.us/p/2ipditb. Install StrawberryPerl instead. Unfortunately,
+# StrawberryPerl not only installs Perl, but also a redundant C/C++ compiler
+# toolchain, and a copy of pkg-config which can cause misdetections for other
+# built products, see
+# https://github.com/StrawberryPerl/Perl-Dist-Strawberry/issues/11 for further
+# details. Remove the redundant and unnecessary parts of the StrawberryPerl
+# install.
+RUN choco install -y strawberryperl && \
+ rmdir /q /s c:\strawberry\c && \
+ del /q c:\strawberry\perl\bin\pkg-config*
+
+# libcxx requires clang(-cl) to be available
+RUN choco install -y sccache llvm
+RUN pip install psutil
+
+RUN curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-ucrt-x86_64.zip && \
+ powershell Expand-Archive llvm-mingw-*-ucrt-x86_64.zip -DestinationPath . && \
+ del llvm-mingw-*-ucrt-x86_64.zip && \
+ ren llvm-mingw-20230320-ucrt-x86_64 llvm-mingw
+
+# configure Python encoding
+ENV PYTHONIOENCODING=UTF-8
+
+# update the path variable
+# C:\Program Files\Git\usr\bin contains a usable bash and other unix tools.
+# C:\llvm-mingw\bin contains Clang configured for mingw targets and
+# corresponding sysroots. Both the 'llvm' package (with Clang defaulting
+# to MSVC targets) and this directory contains executables named
+# 'clang.exe' - add this last to let the other one have precedence.
+# To use these compilers, use the triple prefixed form, e.g.
+# x86_64-w64-mingw32-clang.
+# C:\buildtools and SDK paths are ones that are set by c:\BuildTools\Common7\Tools\VsDevCmd.bat -arch=amd64 -host_arch=amd64
+RUN powershell -Command \
+ [System.Environment]::SetEnvironmentVariable('PATH', \
+ [System.Environment]::GetEnvironmentVariable('PATH', 'machine') + ';C:\Program Files\Git\usr\bin;C:\llvm-mingw\bin' \
+ + ';C:\BuildTools\Common7\IDE\' \
+ + ';C:\BuildTools\Common7\IDE\CommonExt ensions\Microsoft\TeamFoundation\Team Explorer' \
+ + ';C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin' \
+ + ';C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja' \
+ + ';C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\TeamFoundation\Team Explorer' \
+ + ';C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\TestWindow' \
+ + ';C:\BuildTools\Common7\IDE\VC\VCPackages' \
+ + ';C:\BuildTools\Common7\Tools\' \
+ + ';C:\BuildTools\Common7\Tools\devinit' \
+ + ';C:\BuildTools\MSBuild\Current\Bin' \
+ + ';C:\BuildTools\MSBuild\Current\bin\Roslyn' \
+ + ';C:\BuildTools\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64' \
+ + ';C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\x64\' \
+ + ';C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64' \
+ + ';C:\Program Files (x86)\Windows Kits\10\bin\x64' \
+ + ';C:\Windows\Microsoft.NET\Framework64\v4.0.30319' \
+ ,'machine')
+
+# support long file names during git checkout
+RUN git config --system core.longpaths true & \
+ git config --global core.autocrlf false
+
+# handle for debugging of files beeing locked by some processes.
+RUN choco install -y handle
+
+RUN pip3 install pywin32 buildbot-worker==2.8.4
+
+ARG RUNNER_VERSION=2.319.1
+ENV RUNNER_VERSION=$RUNNER_VERSION
+
+RUN powershell -Command \
+ Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v${env:RUNNER_VERSION}/actions-runner-win-x64-${env:RUNNER_VERSION}.zip -OutFile actions-runner-win.zip ; \
+ Add-Type -AssemblyName System.IO.Compression.FileSystem ; \
+ [System.IO.Compression.ZipFile]::ExtractToDirectory('actions-runner-win.zip', $PWD) ;\
+ rm actions-runner-win.zip
>From 91e1371f87cf8fb20a91a3b016a93729c71d2e5f Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Sun, 1 Dec 2024 10:24:25 +0000
Subject: [PATCH 2/6] Try fixing path
---
.github/workflows/build-ci-container-windows.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index 013bc559864fe6..bd14156620efd2 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -27,6 +27,6 @@ jobs:
with:
sparse-checkout: .github/workflows/containers/github-action-ci-windows
- name: Build Container
- working-directory: ./.github/workflows/container/github-action-ci-windows
+ working-directory: .github/workflows/container/github-action-ci-windows
run: |
docker build -t ci-windows-2019 .
>From d246fcb29c6ad5c47680f952753b3ea4df0ce06c Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Sun, 1 Dec 2024 10:25:46 +0000
Subject: [PATCH 3/6] fix typo
---
.github/workflows/build-ci-container-windows.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index bd14156620efd2..38f228ba6fa5a1 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -27,6 +27,6 @@ jobs:
with:
sparse-checkout: .github/workflows/containers/github-action-ci-windows
- name: Build Container
- working-directory: .github/workflows/container/github-action-ci-windows
+ working-directory: .github/workflows/containers/github-action-ci-windows
run: |
docker build -t ci-windows-2019 .
>From 19435b2a7a871fa65068681c3c2cd57bb13c7745 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Wed, 11 Dec 2024 08:52:46 +0000
Subject: [PATCH 4/6] maybe get things working
---
.../workflows/build-ci-container-windows.yml | 45 ++++++++++++++++++-
1 file changed, 44 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index 38f228ba6fa5a1..ce9f96386aca0b 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -21,12 +21,55 @@ jobs:
build-ci-container-windows:
if: github.repository_owner == 'llvm'
runs-on: windows-2019
+ outputs:
+ container-name: ${{ steps.vars.outputs.container-name }}
+ container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
+ container-filename: ${{ steps.vars.outputs.container-filename }}
steps:
- name: Checkout LLVM
uses: actions/checkout at v4
with:
sparse-checkout: .github/workflows/containers/github-action-ci-windows
+ - name: Write Variables
+ id: vars
+ run: |
+ $tag = [int64](Get-Date -UFormat %s)
+ $container_name="ghcr.io/$env:GITHUB_REPOSITORY_OWNER/ci-windows-2019"
+ echo "container-name=$container_name" >> $env:GITHUB_OUTPUT
+ echo "container-name-tag=$container_name:$tag" >> $env:GITHUB_OUTPUT
+ echo "container-filename=ci-windows-$tag.tar" >> $env:GITHUB_OUTPUT
- name: Build Container
working-directory: .github/workflows/containers/github-action-ci-windows
run: |
- docker build -t ci-windows-2019 .
+ docker build -t ${{ steps.vars.outputs.container-name-tag }} .
+ - name: Save container image
+ run: |
+ podman save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }}
+ - name: Upload container image
+ uses: actions/upload-artifact at v4
+ with:
+ name: container
+ path: ${{ steps.vars.outputs.container-filename }}
+ retention-days: 14
+
+ push-ci-container:
+ if: github.event_name == 'push'
+ needs:
+ - build-ci-container-windows
+ permissions:
+ packages: write
+ runs-on: windows-2019
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - name: Download container
+ uses: actions/download-artifact at v4
+ with:
+ name: container
+ - name: Push Container
+ run: |
+ docker load -i ${{ needs.build-ci-container.outptus.container-filename }}
+ docker tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest
+ docker login -u ${{ github.actor }} -p $env:GITHUB_TOKEN ghcr.io
+ docker push ${{ needs.build-ci-container.outputs.container-name-tag }}
+ docker push ${{ needs.build-ci-container.outputs.container-name }}:latest
>From 417ac5fd6bd6444f2c79e9e765a5210a0b586f98 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Wed, 11 Dec 2024 08:54:17 +0000
Subject: [PATCH 5/6] Try delimiting names
---
.github/workflows/build-ci-container-windows.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index ce9f96386aca0b..a89272e5f37bb4 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -35,9 +35,9 @@ jobs:
run: |
$tag = [int64](Get-Date -UFormat %s)
$container_name="ghcr.io/$env:GITHUB_REPOSITORY_OWNER/ci-windows-2019"
- echo "container-name=$container_name" >> $env:GITHUB_OUTPUT
- echo "container-name-tag=$container_name:$tag" >> $env:GITHUB_OUTPUT
- echo "container-filename=ci-windows-$tag.tar" >> $env:GITHUB_OUTPUT
+ echo "container-name=${container_name}" >> $env:GITHUB_OUTPUT
+ echo "container-name-tag=${container_name}:${tag}" >> $env:GITHUB_OUTPUT
+ echo "container-filename=ci-windows-${tag}.tar" >> $env:GITHUB_OUTPUT
- name: Build Container
working-directory: .github/workflows/containers/github-action-ci-windows
run: |
>From cd35ef2651aa8fa8140aad8e00f6feba6025db20 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman at google.com>
Date: Wed, 11 Dec 2024 09:22:24 +0000
Subject: [PATCH 6/6] podman -> docker
---
.github/workflows/build-ci-container-windows.yml | 2 +-
.github/workflows/build-ci-container.yml | 10 +++-------
2 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index a89272e5f37bb4..898cdce097e97e 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -44,7 +44,7 @@ jobs:
docker build -t ${{ steps.vars.outputs.container-name-tag }} .
- name: Save container image
run: |
- podman save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }}
+ docker save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }}
- name: Upload container image
uses: actions/upload-artifact at v4
with:
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index f037a91f6e5d01..c9f225a7c28358 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -20,7 +20,9 @@ on:
jobs:
build-ci-container:
if: github.repository_owner == 'llvm'
- runs-on: depot-ubuntu-22.04-16
+ runs-on: llvm-premerge-linux-runners
+ container:
+ image: gcr.io/kaniko-project/executor:debug
outputs:
container-name: ${{ steps.vars.outputs.container-name }}
container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
@@ -57,12 +59,6 @@ jobs:
path: ${{ steps.vars.outputs.container-filename }}
retention-days: 14
- - name: Test Container
- run: |
- for image in ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}; do
- podman run --rm -it $image /usr/bin/bash -x -c 'printf '\''#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello'
- done
-
push-ci-container:
if: github.event_name == 'push'
needs:
More information about the llvm-commits
mailing list