[llvm] [Github] Fetch all commits in PR for code formatting checks (PR #69766)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 28 18:15:37 PDT 2023
https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/69766
>From f44998a0ee126e3cfcd881d3351deb41c958f77b Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Fri, 20 Oct 2023 14:23:06 -0700
Subject: [PATCH 01/11] [Github] Fetch all commits in PR for code formatting
checks
This patch ensures that all commits from a PR are fetched before running
the rest of the job. This allows the job to scale to any number of
commits per PR. In addition, this significantly reduces the time taken
in the files-changed step where the job would itself try to unshallow
the clone, but do so in some inefficient way. That step which previously
took ~10 minutes now takes 3 seconds after this patch.
---
.github/workflows/pr-code-format.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 3a91ffb0b1ad9a7..8ca27c2f6363b16 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -10,14 +10,13 @@ jobs:
- name: Fetch LLVM sources
uses: actions/checkout at v4
with:
- fetch-depth: 2
+ fetch-depth: ${{ github.event.pull_request.commits }}
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files at v39
with:
separator: ","
- fetch_depth: 100 # Fetches only the last 10 commits
- name: "Listed files"
run: |
>From eec1483d5ce51f5593edff7936a4e14f2393d3ad Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 16:44:11 -0700
Subject: [PATCH 02/11] Change checkout depth and move changed files location
---
.github/workflows/pr-code-format.yml | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 8ca27c2f6363b16..727f4dff40de52f 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -7,17 +7,24 @@ jobs:
code_formatter:
runs-on: ubuntu-latest
steps:
- - name: Fetch LLVM sources
- uses: actions/checkout at v4
- with:
- fetch-depth: ${{ github.event.pull_request.commits }}
-
+ # Get changed files before checking out the repository to force the action
+ # to analyze the diff from the Github API rather than looking at the
+ # shallow clone and erroring out, which is significantly more prone to
+ # failure.
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files at v39
with:
separator: ","
+ - name: Calculate number of commits to fetch (PR)
+ run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
+
+ - name: Fetch LLVM sources
+ uses: actions/checkout at v4
+ with:
+ fetch-depth: ${{ env.PR_FETCH_DEPTH }}
+
- name: "Listed files"
run: |
echo "Formatting files:"
>From 3b00d938e9985a395ed405e24e3038d4df52aa14 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 17:07:09 -0700
Subject: [PATCH 03/11] Add some testing stuff
---
.github/workflows/pr-code-format.yml | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 727f4dff40de52f..fb5a5ac5ffdf27b 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -30,6 +30,10 @@ jobs:
echo "Formatting files:"
echo "${{ steps.changed-files.outputs.all_changed_files }}"
+ - name: "Git log"
+ run: |
+ git log
+
- name: Install clang-format
uses: aminya/setup-cpp at v1
with:
@@ -58,3 +62,11 @@ jobs:
--start-rev $START_REV \
--end-rev $END_REV \
--changed-files "$CHANGED_FILES"
+ - name: Test thing
+ env:
+ GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+ START_REV: ${{ github.event.pull_request.base.sha }}
+ END_REV: ${{ github.event.pull_request.head.sha }}
+ CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+ run: |
+ git-clang-format --diff $START_REV $END_REV -- $CHANGED_FILES
>From 57cc720e30b913f9185ef0169c3be1c0100dccf8 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 17:12:17 -0700
Subject: [PATCH 04/11] Update ref
---
.github/workflows/pr-code-format.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index fb5a5ac5ffdf27b..9668fc46507eba8 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -23,6 +23,7 @@ jobs:
- name: Fetch LLVM sources
uses: actions/checkout at v4
with:
+ ref: ${{ github.ref }}
fetch-depth: ${{ env.PR_FETCH_DEPTH }}
- name: "Listed files"
>From 9127b9945249aa4d43eaba4c48e111532c543e60 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 17:22:31 -0700
Subject: [PATCH 05/11] try changing ref
---
.github/workflows/pr-code-format.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 9668fc46507eba8..80e4717d22f61f3 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -23,7 +23,7 @@ jobs:
- name: Fetch LLVM sources
uses: actions/checkout at v4
with:
- ref: ${{ github.ref }}
+ ref: ${{ github.event.pull_request.head.ref }}
fetch-depth: ${{ env.PR_FETCH_DEPTH }}
- name: "Listed files"
>From bf06eb19c5fd213aaa187b0937228427ed70991a Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 17:38:41 -0700
Subject: [PATCH 06/11] Try another thing
---
.github/workflows/pr-code-format.yml | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 80e4717d22f61f3..21e88692f751ac6 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -17,14 +17,11 @@ jobs:
with:
separator: ","
- - name: Calculate number of commits to fetch (PR)
- run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
-
- name: Fetch LLVM sources
uses: actions/checkout at v4
with:
ref: ${{ github.event.pull_request.head.ref }}
- fetch-depth: ${{ env.PR_FETCH_DEPTH }}
+ fetch-depth: ${{ github.event.pull_request.commits }}
- name: "Listed files"
run: |
@@ -53,21 +50,19 @@ jobs:
- name: Run code formatter
env:
GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
- START_REV: ${{ github.event.pull_request.base.sha }}
- END_REV: ${{ github.event.pull_request.head.sha }}
+ PR_DEPTH: ${{ github.event.pull_request.commits }}
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
run: |
python llvm/utils/git/code-format-helper.py \
--token ${{ secrets.GITHUB_TOKEN }} \
--issue-number $GITHUB_PR_NUMBER \
- --start-rev $START_REV \
- --end-rev $END_REV \
+ --start-rev HEAD~$PR_DEPTH \
+ --end-rev HEAD \
--changed-files "$CHANGED_FILES"
- name: Test thing
env:
GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
- START_REV: ${{ github.event.pull_request.base.sha }}
- END_REV: ${{ github.event.pull_request.head.sha }}
+ PR_DEPTH: ${{ github.event.pull_request.commits }}
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
run: |
- git-clang-format --diff $START_REV $END_REV -- $CHANGED_FILES
+ git-clang-format --diff HEAD~$PR_DEPTH HEAD -- $CHANGED_FILES
>From 6ea1e85d50c4dd03c93485f4200d7872e944c796 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 17:42:51 -0700
Subject: [PATCH 07/11] More testing
---
.github/workflows/pr-code-format.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 21e88692f751ac6..a7479132128c17d 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -65,4 +65,4 @@ jobs:
PR_DEPTH: ${{ github.event.pull_request.commits }}
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
run: |
- git-clang-format --diff HEAD~$PR_DEPTH HEAD -- $CHANGED_FILES
+ git-clang-format --diff HEAD~1 HEAD -- $CHANGED_FILES
>From 003b656b8edcfb72ed2d500d4616507f8e50329c Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 17:49:22 -0700
Subject: [PATCH 08/11] More testing
---
.github/workflows/pr-code-format.yml | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index a7479132128c17d..66a8499088c5db0 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -17,11 +17,14 @@ jobs:
with:
separator: ","
+ - name: Calculate number of commits to fetch (PR)
+ run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
+
- name: Fetch LLVM sources
uses: actions/checkout at v4
with:
ref: ${{ github.event.pull_request.head.ref }}
- fetch-depth: ${{ github.event.pull_request.commits }}
+ fetch-depth: ${{ env.PR_DEPTH }}
- name: "Listed files"
run: |
@@ -65,4 +68,4 @@ jobs:
PR_DEPTH: ${{ github.event.pull_request.commits }}
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
run: |
- git-clang-format --diff HEAD~1 HEAD -- $CHANGED_FILES
+ git-clang-format --diff HEAD~$PR_DEPTH HEAD -- $CHANGED_FILES
>From c35e9cb200251c0cdf49a880efcf09de11674449 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 17:52:36 -0700
Subject: [PATCH 09/11] fix typo
---
.github/workflows/pr-code-format.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 66a8499088c5db0..4757443760d1ba2 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -24,7 +24,7 @@ jobs:
uses: actions/checkout at v4
with:
ref: ${{ github.event.pull_request.head.ref }}
- fetch-depth: ${{ env.PR_DEPTH }}
+ fetch-depth: ${{ env.PR_FETCH_DEPTH }}
- name: "Listed files"
run: |
>From 1f68c9ccf305eb10ce019a50ac4403ed117b410c Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 18:13:43 -0700
Subject: [PATCH 10/11] Switch some things around
---
.github/workflows/pr-code-format.yml | 33 +++++++++++++++-------------
1 file changed, 18 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 4757443760d1ba2..8b0a31ae9ed36e9 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -17,24 +17,33 @@ jobs:
with:
separator: ","
- - name: Calculate number of commits to fetch (PR)
+ - name: Calculate number of commits to fetch
run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
- - name: Fetch LLVM sources
+ - name: Fetch PR sources
uses: actions/checkout at v4
with:
ref: ${{ github.event.pull_request.head.ref }}
fetch-depth: ${{ env.PR_FETCH_DEPTH }}
+ path: pr-sources
+
+ # We need to make sure that we aren't executing/using any code from the
+ # PR for security reasons as we're using pull_request_target. Checkout
+ # the target branch with the necessary files.
+ - name: Fetch LLVM Sources
+ uses: actions/checkout at v4
+ with:
+ sparse-checkout: |
+ llvm/utils/git/requirements_formatting.txt
+ llvm/utils/git/code-format-helper.py
+ sparse-checkout-cone-mode: false
+ path: llvm-sources
- name: "Listed files"
run: |
echo "Formatting files:"
echo "${{ steps.changed-files.outputs.all_changed_files }}"
- - name: "Git log"
- run: |
- git log
-
- name: Install clang-format
uses: aminya/setup-cpp at v1
with:
@@ -45,16 +54,17 @@ jobs:
with:
python-version: '3.11'
cache: 'pip'
- cache-dependency-path: 'llvm/utils/git/requirements_formatting.txt'
+ cache-dependency-path: 'llvm-sources/llvm/utils/git/requirements_formatting.txt'
- name: Install python dependencies
- run: pip install -r llvm/utils/git/requirements_formatting.txt
+ run: pip install -r llvm-sources/llvm/utils/git/requirements_formatting.txt
- name: Run code formatter
env:
GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
PR_DEPTH: ${{ github.event.pull_request.commits }}
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+ working_directory: ./llvm-sources
run: |
python llvm/utils/git/code-format-helper.py \
--token ${{ secrets.GITHUB_TOKEN }} \
@@ -62,10 +72,3 @@ jobs:
--start-rev HEAD~$PR_DEPTH \
--end-rev HEAD \
--changed-files "$CHANGED_FILES"
- - name: Test thing
- env:
- GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
- PR_DEPTH: ${{ github.event.pull_request.commits }}
- CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
- run: |
- git-clang-format --diff HEAD~$PR_DEPTH HEAD -- $CHANGED_FILES
>From e65a4e8cf9b0281f0f93869825bcdaa0b3ac79e6 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sat, 28 Oct 2023 18:15:21 -0700
Subject: [PATCH 11/11] Fix typo
---
.github/workflows/pr-code-format.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 8b0a31ae9ed36e9..2b0a9975cc320bf 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -64,7 +64,7 @@ jobs:
GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
PR_DEPTH: ${{ github.event.pull_request.commits }}
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
- working_directory: ./llvm-sources
+ working-directory: ./llvm-sources
run: |
python llvm/utils/git/code-format-helper.py \
--token ${{ secrets.GITHUB_TOKEN }} \
More information about the llvm-commits
mailing list