[libcxx-commits] [libcxx] [llvm] [libc++] [WIP] Add metrics collection for libc++ premerge testing. (PR #152801)
via libcxx-commits
libcxx-commits at lists.llvm.org
Thu Aug 21 08:56:08 PDT 2025
https://github.com/cmtice updated https://github.com/llvm/llvm-project/pull/152801
>From 67142b91ab1cfb76de821852316a687e0303cd26 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 8 Aug 2025 14:27:28 -0700
Subject: [PATCH 01/13] [libc++] [WIP] Add metrics collection for libc++
premerge testing.
** DRAFT!! DO NOT REVIEW! DO NOT MERGE! **
Work-in-progress, playing around with adding metrics collection
and dashboard for libc++ premerge testing data.
---
.github/workflows/libcxx-build-and-test.yaml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 4b29c9296bf9d..739e56e56f459 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -35,6 +35,7 @@ concurrency:
jobs:
stage1:
+ name: libc++ Stage1 Testing
if: github.repository_owner == 'llvm'
runs-on: llvm-premerge-libcxx-runners
continue-on-error: false
@@ -72,6 +73,7 @@ jobs:
**/CMakeOutput.log
**/crash_diagnostics/*
stage2:
+ name: libc++ Stage2 Testing
if: github.repository_owner == 'llvm'
runs-on: llvm-premerge-libcxx-runners
needs: [ stage1 ]
@@ -117,6 +119,7 @@ jobs:
**/CMakeOutput.log
**/crash_diagnostics/*
stage3:
+ name: libc++ Stage3 Testing
if: github.repository_owner == 'llvm'
needs: [ stage2 ]
continue-on-error: false
>From 46114b04fe54f4b3c4c2e9b15dab64f8e4b95686 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 8 Aug 2025 14:52:02 -0700
Subject: [PATCH 02/13] Update premerge metrics to collect metrics for libc++ 3
stages.
---
.ci/metrics/metrics.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index 26fdeef1913ab..ec762f65bf85c 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -28,7 +28,10 @@
# Lists the Github workflows we want to track. Maps the Github job name to
# the metric name prefix in grafana.
# This metric name is also used as a key in the job->name map.
-GITHUB_WORKFLOW_TO_TRACK = {"CI Checks": "github_llvm_premerge_checks"}
+GITHUB_WORKFLOW_TO_TRACK = {
+ "CI Checks": "github_llvm_premerge_checks",
+ "Build and Test libc++": "github_libc++_premerge_checks",
+}
# Lists the Github jobs to track for a given workflow. The key is the stable
# name (metric name) of the workflow (see GITHUB_WORKFLOW_TO_TRACK).
@@ -39,6 +42,11 @@
"Build and Test Linux": "premerge_linux",
"Build and Test Windows": "premerge_windows",
}
+ "github_libc++_premerge_checks": {
+ "libc++ Stage1 Testing": "premerge_libcxx_stage1",
+ "libc++ Stage2 Testing": "premerge_libcxx_stage2",
+ "libc++ Stage3 Testing": "premerge_libcxx_stage3",
+ }
}
# The number of workflows to pull when sampling Github workflows.
>From 95729ff959682292b882d179109b910229ba77fe Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 8 Aug 2025 15:03:30 -0700
Subject: [PATCH 03/13] Add missing commas.
---
.ci/metrics/metrics.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index ec762f65bf85c..07f958f06f729 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -41,12 +41,12 @@
"github_llvm_premerge_checks": {
"Build and Test Linux": "premerge_linux",
"Build and Test Windows": "premerge_windows",
- }
+ },
"github_libc++_premerge_checks": {
"libc++ Stage1 Testing": "premerge_libcxx_stage1",
"libc++ Stage2 Testing": "premerge_libcxx_stage2",
"libc++ Stage3 Testing": "premerge_libcxx_stage3",
- }
+ },
}
# The number of workflows to pull when sampling Github workflows.
>From f564525150fc80883dbc7cf8d53cebfddf3678fe Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 15 Aug 2025 10:21:11 -0700
Subject: [PATCH 04/13] Update PR to trigger premerge testing.
---
libcxx/docs/CarolinesTestDoc.txt | 6 ++++++
1 file changed, 6 insertions(+)
create mode 100644 libcxx/docs/CarolinesTestDoc.txt
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
new file mode 100644
index 0000000000000..0ee5b29d5c291
--- /dev/null
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -0,0 +1,6 @@
+This is a nonsense document, created for the sole purpose of
+triggering libcxx premerge testing (so I can test metrics collection).
+
+
+This is NOT intended to ever be reviewed or merged or committed. If it gets
+accidentally committed, please remove it asap.
>From 6ddc381a6d00c818a243f1ac787eaec5914d99a1 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 15 Aug 2025 15:04:50 -0700
Subject: [PATCH 05/13] Prod PR for testing.
---
libcxx/docs/CarolinesTestDoc.txt | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index 0ee5b29d5c291..8525f5c646c9d 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -1,6 +1,8 @@
This is a nonsense document, created for the sole purpose of
triggering libcxx premerge testing (so I can test metrics collection).
-
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
+
+Sample text.
+
>From 0bf2562b8313b4cbe82d4caf84b640bee13c686b Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Mon, 18 Aug 2025 11:15:49 -0700
Subject: [PATCH 06/13] Poke PR to kick off testing.
---
libcxx/docs/CarolinesTestDoc.txt | 2 --
1 file changed, 2 deletions(-)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index 8525f5c646c9d..36b255be72237 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -4,5 +4,3 @@ triggering libcxx premerge testing (so I can test metrics collection).
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
-Sample text.
-
>From 65b0c06039c3bdffe357e8610f1f9bb8ff12b3c3 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Mon, 18 Aug 2025 14:44:00 -0700
Subject: [PATCH 07/13] Poke PR.
---
libcxx/docs/CarolinesTestDoc.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index 36b255be72237..bf977fc515e2c 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -4,3 +4,4 @@ triggering libcxx premerge testing (so I can test metrics collection).
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
+Sample text goes here.
>From 09d22f42d876897e963229b3f63bf47110bbfba3 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Wed, 20 Aug 2025 07:58:24 -0700
Subject: [PATCH 08/13] Remove unnecessary name change.
---
.github/workflows/libcxx-build-and-test.yaml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 739e56e56f459..4b29c9296bf9d 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -35,7 +35,6 @@ concurrency:
jobs:
stage1:
- name: libc++ Stage1 Testing
if: github.repository_owner == 'llvm'
runs-on: llvm-premerge-libcxx-runners
continue-on-error: false
@@ -73,7 +72,6 @@ jobs:
**/CMakeOutput.log
**/crash_diagnostics/*
stage2:
- name: libc++ Stage2 Testing
if: github.repository_owner == 'llvm'
runs-on: llvm-premerge-libcxx-runners
needs: [ stage1 ]
@@ -119,7 +117,6 @@ jobs:
**/CMakeOutput.log
**/crash_diagnostics/*
stage3:
- name: libc++ Stage3 Testing
if: github.repository_owner == 'llvm'
needs: [ stage2 ]
continue-on-error: false
>From 5a930115b0fee56a3be85ee1f1de7f4ca57e7034 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Wed, 20 Aug 2025 10:57:02 -0700
Subject: [PATCH 09/13] Poke PR.
---
libcxx/docs/CarolinesTestDoc.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index bf977fc515e2c..e34bec83d623b 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -4,4 +4,4 @@ triggering libcxx premerge testing (so I can test metrics collection).
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
-Sample text goes here.
+Sample text goes here. And more sample text.
>From 1c405075ede64311181f5d12159b1fc4f16babad Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Wed, 20 Aug 2025 13:37:42 -0700
Subject: [PATCH 10/13] Poke PR.
---
libcxx/docs/CarolinesTestDoc.txt | 1 -
1 file changed, 1 deletion(-)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index e34bec83d623b..36b255be72237 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -4,4 +4,3 @@ triggering libcxx premerge testing (so I can test metrics collection).
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
-Sample text goes here. And more sample text.
>From 2a32dfb66a298e1712d528ee87b7e238cbf7cf37 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Thu, 21 Aug 2025 08:36:44 -0700
Subject: [PATCH 11/13] Remove fake testing doc.
---
libcxx/docs/CarolinesTestDoc.txt | 6 ------
1 file changed, 6 deletions(-)
delete mode 100644 libcxx/docs/CarolinesTestDoc.txt
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
deleted file mode 100644
index 36b255be72237..0000000000000
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-This is a nonsense document, created for the sole purpose of
-triggering libcxx premerge testing (so I can test metrics collection).
-
-This is NOT intended to ever be reviewed or merged or committed. If it gets
-accidentally committed, please remove it asap.
-
>From c8c42b7697a17e2a374e27808397ea0a5bb8d7c6 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Thu, 21 Aug 2025 08:39:37 -0700
Subject: [PATCH 12/13] Add fake test document, for poking PR testing.
---
libcxx/docs/CarolineTest.txt | 4 ++++
1 file changed, 4 insertions(+)
create mode 100644 libcxx/docs/CarolineTest.txt
diff --git a/libcxx/docs/CarolineTest.txt b/libcxx/docs/CarolineTest.txt
new file mode 100644
index 0000000000000..6eaa0fcd3ca11
--- /dev/null
+++ b/libcxx/docs/CarolineTest.txt
@@ -0,0 +1,4 @@
+This is another test document.
+
+Please delete this document before merging any PRs; this document is NOT
+meant to be merged upstream.
>From 61ac17999b0f2358d2a1ee30e2bcf030938717e3 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Thu, 21 Aug 2025 08:54:49 -0700
Subject: [PATCH 13/13] Remove fake document (used for poking test runs).
Finish updating metrics.py to collect libc++ testing metrics.
---
.ci/metrics/metrics.py | 178 ++++++++++++++++++++++++++++++++---
libcxx/docs/CarolineTest.txt | 4 -
2 files changed, 167 insertions(+), 15 deletions(-)
delete mode 100644 libcxx/docs/CarolineTest.txt
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index 07f958f06f729..f3b7aab2eb53b 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -30,7 +30,7 @@
# This metric name is also used as a key in the job->name map.
GITHUB_WORKFLOW_TO_TRACK = {
"CI Checks": "github_llvm_premerge_checks",
- "Build and Test libc++": "github_libc++_premerge_checks",
+ "Build and Test libc++": "github_libcxx_premerge_checks",
}
# Lists the Github jobs to track for a given workflow. The key is the stable
@@ -42,10 +42,10 @@
"Build and Test Linux": "premerge_linux",
"Build and Test Windows": "premerge_windows",
},
- "github_libc++_premerge_checks": {
- "libc++ Stage1 Testing": "premerge_libcxx_stage1",
- "libc++ Stage2 Testing": "premerge_libcxx_stage2",
- "libc++ Stage3 Testing": "premerge_libcxx_stage3",
+ "github_libcxx_premerge_checks": {
+ "stage1": "premerge_libcxx_stage1",
+ "stage2": "premerge_libcxx_stage2",
+ "stage3": "premerge_libcxx_stage3",
},
}
@@ -70,13 +70,14 @@
# by trial and error).
GRAFANA_METRIC_MAX_AGE_MN = 120
-
@dataclass
class JobMetrics:
job_name: str
queue_time: int
run_time: int
status: int
+ created_at_ns: int
+ started_at_ns: int
completed_at_ns: int
workflow_id: int
workflow_name: str
@@ -89,6 +90,139 @@ class GaugeMetric:
time_ns: int
+ at dataclass
+class AggregateMetric:
+ aggregate_name: str
+ aggregate_queue_time: int
+ aggregate_run_time: int
+ aggregate_status: int
+ workflow_id: int
+
+
+def create_and_append_libcxx_aggregates(
+ workflow_metrics: list[JobMetrics]) -> list[JobMetrics,AggregateMetric]:
+ """
+ Find libc++ JobMetric entries and create aggregate metrics for them.
+
+ Sort the libc++ JobMetric entries by workflow id, and for each workflow
+ id group them by stages. Create an aggreate metric for each stage for each
+ unique workflow id. Append each aggregate metric to the workflow_metrics
+ list.
+
+ How aggreates are computed:
+ queue time: Time from when first job in group is created until last job in
+ group has started.
+ run time: Time from when first job in group starts running until last job
+ in group finishes running.
+ status: logical 'or' of all the job statuses in the group.
+ """
+ # Separate the jobs by workflow_id. Only look at JobMetrics entries.
+ aggregate_data = dict()
+ for job in workflow_metrics:
+ # Only want to look at JobMetrics
+ if not isinstance(job, JobMetrics):
+ continue
+ # Only want libc++ jobs.
+ if job.workflow_name != "Build and Test libc++":
+ continue
+ if job.workflow_id not in aggregate_data.keys():
+ aggregate_data[job.workflow_id] = [ job ]
+ else:
+ aggregate_data[job.workflow_id].append(job)
+
+ # Go through each aggregate_data list (workflow id) and find all the
+ # needed data
+ for ag_workflow_id in aggregate_data:
+ job_list = aggregate_data[ag_workflow_id]
+ stage1_jobs = list()
+ stage2_jobs = list()
+ stage3_jobs = list()
+ # sort jobs into stage1, stage2, & stage3.
+ for job in job_list:
+ if job.job_name.find('stage1') > 0:
+ stage1_jobs.append(job)
+ elif job.job_name.find('stage2') > 0:
+ stage2_jobs.append(job)
+ elif job.job_name.find('stage3') > 0:
+ stage3_jobs.append(job)
+
+ for job_list in [ stage1_jobs, stage2_jobs, stage3_jobs]:
+ if len(job_list) < 1:
+ # No jobs in that stage this time around.
+ continue
+
+ # Get the aggregate name.
+ ag_name = "github_libcxx_premerge_checks_"
+ if job_list[0].job_name.find('stage1') > 0:
+ ag_name = ag_name + "stage1_aggregate"
+ elif job_list[0].job_name.find('stage2') > 0:
+ ag_name = ag_name + "stage2_aggregate"
+ elif job_list[0].job_name.find('stage3') > 0:
+ ag_name = ag_name + "stage3_aggregate"
+ else:
+ ag_name = ag_name + "unknown_aggregate"
+
+ # Initialize the rest of the aggregate values
+ earliest_create = job_list[0].created_at_ns
+ earliest_start = job_list[0].started_at_ns
+ earliest_complete = job_list[0].completed_at_ns
+ latest_start = job_list[0].started_at_ns
+ latest_complete = job_list[0].completed_at_ns
+ ag_status = job_list[0].status
+
+ # Go through rest of jobs for this workflow id, updating stats
+ for job in job_list[1:]:
+ # Update the status
+ ag_status = ag_status or job.status
+ # Get the earliest & latest times
+ if job.created_at_ns < earliest_create:
+ earliest_create = job.created_at_ns
+ if job.completed_at_ns < earliest_complete:
+ earliest_complete = job.completed_at_ns
+ if job.started_at_ns > latest_start:
+ latest_start = job.started_at_ns
+ if job.started_at_ns < earliest_start:
+ earliest_start = job.started_at_ns
+ if job.completed_at_ns > latest_complete:
+ latest_complete = job.completed_at_ns
+
+ # Compute aggregate run time (in seconds, not ns)
+ ag_run_time = (latest_complete - earliest_start) / 1000000000
+ # Compute aggregate queue time (in seconds, not ns)
+ ag_queue_time = (latest_start - earliest_create) / 1000000000
+ # Append the aggregate metrics to the workflow metrics list.
+ workflow_metrics.append(
+ AggregateMetric(
+ ag_name, ag_queue_time, ag_run_time, ag_status,
+ ag_workflow_id
+ )
+ )
+ return
+
+def clean_up_libcxx_job_name(old_name: str) -> str:
+ """
+ Convert libcxx job names to generically legal strings.
+
+ Take a name like 'stage1 (generic-cxx03, clang-22, clang++-22)'
+ and convert it to 'stage1_generic_cxx03__clang_22__clangxx_22'.
+ (Remove parentheses; replace commas, hyphens and spaces with
+ underscores; replace '+' with 'x'.
+ """
+ # Names should have exactly one set of parentheses, so break on that. If
+ # they don't have any parentheses, then don't update them at all.
+ if old_name.find('(') == -1:
+ return old_name
+ stage, remainder = old_name.split('(')
+ stage = stage.strip()
+ if remainder[-1] == ')':
+ remainder = remainder[:-1]
+ remainder = remainder.replace('-', '_')
+ remainder = remainder.replace(',', '_')
+ remainder = remainder.replace(' ', '_')
+ remainder = remainder.replace('+', 'x')
+ new_name = stage + '_' + remainder
+ return new_name
+
def github_get_metrics(
github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
) -> tuple[list[JobMetrics], int]:
@@ -151,9 +285,14 @@ def github_get_metrics(
break
# This workflow is not interesting to us.
- if task.name not in GITHUB_WORKFLOW_TO_TRACK:
+ if (task.name not in GITHUB_WORKFLOW_TO_TRACK
+ and task.name != "Build and Test libc++"):
continue
+ libcxx_testing = False
+ if task.name == "Build and Test libc++":
+ libcxx_testing = True
+
if task.status == "completed":
workflow_seen_as_completed.add(task.id)
@@ -163,11 +302,20 @@ def github_get_metrics(
name_prefix = GITHUB_WORKFLOW_TO_TRACK[task.name]
for job in task.jobs():
+ if libcxx_testing:
+ # We're not running macos or windows libc++ tests on our
+ # infrastructure.
+ if (job.name.find("macos") != -1 or
+ job.name.find("windows") != -1):
+ continue
# This job is not interesting to us.
- if job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
+ elif job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
continue
- name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
+ if libcxx_testing:
+ name_suffix = clean_up_libcxx_job_name(job.name)
+ else:
+ name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
metric_name = name_prefix + "_" + name_suffix
if task.status != "completed":
@@ -216,8 +364,10 @@ def github_get_metrics(
continue
logging.info(f"Adding a job metric for job {job.id} in workflow {task.id}")
- # The timestamp associated with the event is expected by Grafana to be
- # in nanoseconds.
+ # The timestamp associated with the event is expected by Grafana to
+ # be in nanoseconds.
+ created_at_ns = int(created_at.timestamp()) * 10**9
+ started_at_ns = int(started_at.timestamp()) * 10**9
completed_at_ns = int(completed_at.timestamp()) * 10**9
workflow_metrics.append(
JobMetrics(
@@ -225,12 +375,18 @@ def github_get_metrics(
queue_time.seconds,
run_time.seconds,
job_result,
+ created_at_ns,
+ started_at_ns,
completed_at_ns,
task.id,
task.name,
)
)
+ # Finished collecting the JobMetrics for all jobs; now create the
+ # aggregates for any libc++ jobs.
+ create_and_append_libcxx_aggregates(workflow_metrics)
+
for name, value in queued_count.items():
workflow_metrics.append(
GaugeMetric(f"workflow_queue_size_{name}", value, time.time_ns())
diff --git a/libcxx/docs/CarolineTest.txt b/libcxx/docs/CarolineTest.txt
deleted file mode 100644
index 6eaa0fcd3ca11..0000000000000
--- a/libcxx/docs/CarolineTest.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-This is another test document.
-
-Please delete this document before merging any PRs; this document is NOT
-meant to be merged upstream.
More information about the libcxx-commits
mailing list