[libcxx-commits] [libcxx] [llvm] [llvm][CI] Add metrics collection for libc++ premerge testing. (PR #152801)
via libcxx-commits
libcxx-commits at lists.llvm.org
Fri Aug 22 10:36:21 PDT 2025
https://github.com/cmtice updated https://github.com/llvm/llvm-project/pull/152801
>From 67142b91ab1cfb76de821852316a687e0303cd26 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 8 Aug 2025 14:27:28 -0700
Subject: [PATCH 01/15] [libc++] [WIP] Add metrics collection for libc++
premerge testing.
** DRAFT!! DO NOT REVIEW! DO NOT MERGE! **
Work-in-progress, playing around with adding metrics collection
and dashboard for libc++ premerge testing data.
---
.github/workflows/libcxx-build-and-test.yaml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 4b29c9296bf9d..739e56e56f459 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -35,6 +35,7 @@ concurrency:
jobs:
stage1:
+ name: libc++ Stage1 Testing
if: github.repository_owner == 'llvm'
runs-on: llvm-premerge-libcxx-runners
continue-on-error: false
@@ -72,6 +73,7 @@ jobs:
**/CMakeOutput.log
**/crash_diagnostics/*
stage2:
+ name: libc++ Stage2 Testing
if: github.repository_owner == 'llvm'
runs-on: llvm-premerge-libcxx-runners
needs: [ stage1 ]
@@ -117,6 +119,7 @@ jobs:
**/CMakeOutput.log
**/crash_diagnostics/*
stage3:
+ name: libc++ Stage3 Testing
if: github.repository_owner == 'llvm'
needs: [ stage2 ]
continue-on-error: false
>From 46114b04fe54f4b3c4c2e9b15dab64f8e4b95686 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 8 Aug 2025 14:52:02 -0700
Subject: [PATCH 02/15] Update premerge metrics to collect metrics for libc++ 3
stages.
---
.ci/metrics/metrics.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index 26fdeef1913ab..ec762f65bf85c 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -28,7 +28,10 @@
# Lists the Github workflows we want to track. Maps the Github job name to
# the metric name prefix in grafana.
# This metric name is also used as a key in the job->name map.
-GITHUB_WORKFLOW_TO_TRACK = {"CI Checks": "github_llvm_premerge_checks"}
+GITHUB_WORKFLOW_TO_TRACK = {
+ "CI Checks": "github_llvm_premerge_checks",
+ "Build and Test libc++": "github_libc++_premerge_checks",
+}
# Lists the Github jobs to track for a given workflow. The key is the stable
# name (metric name) of the workflow (see GITHUB_WORKFLOW_TO_TRACK).
@@ -39,6 +42,11 @@
"Build and Test Linux": "premerge_linux",
"Build and Test Windows": "premerge_windows",
}
+ "github_libc++_premerge_checks": {
+ "libc++ Stage1 Testing": "premerge_libcxx_stage1",
+ "libc++ Stage2 Testing": "premerge_libcxx_stage2",
+ "libc++ Stage3 Testing": "premerge_libcxx_stage3",
+ }
}
# The number of workflows to pull when sampling Github workflows.
>From 95729ff959682292b882d179109b910229ba77fe Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 8 Aug 2025 15:03:30 -0700
Subject: [PATCH 03/15] Add missing commas.
---
.ci/metrics/metrics.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index ec762f65bf85c..07f958f06f729 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -41,12 +41,12 @@
"github_llvm_premerge_checks": {
"Build and Test Linux": "premerge_linux",
"Build and Test Windows": "premerge_windows",
- }
+ },
"github_libc++_premerge_checks": {
"libc++ Stage1 Testing": "premerge_libcxx_stage1",
"libc++ Stage2 Testing": "premerge_libcxx_stage2",
"libc++ Stage3 Testing": "premerge_libcxx_stage3",
- }
+ },
}
# The number of workflows to pull when sampling Github workflows.
>From f564525150fc80883dbc7cf8d53cebfddf3678fe Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 15 Aug 2025 10:21:11 -0700
Subject: [PATCH 04/15] Update PR to trigger premerge testing.
---
libcxx/docs/CarolinesTestDoc.txt | 6 ++++++
1 file changed, 6 insertions(+)
create mode 100644 libcxx/docs/CarolinesTestDoc.txt
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
new file mode 100644
index 0000000000000..0ee5b29d5c291
--- /dev/null
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -0,0 +1,6 @@
+This is a nonsense document, created for the sole purpose of
+triggering libcxx premerge testing (so I can test metrics collection).
+
+
+This is NOT intended to ever be reviewed or merged or committed. If it gets
+accidentally committed, please remove it asap.
>From 6ddc381a6d00c818a243f1ac787eaec5914d99a1 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 15 Aug 2025 15:04:50 -0700
Subject: [PATCH 05/15] Prod PR for testing.
---
libcxx/docs/CarolinesTestDoc.txt | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index 0ee5b29d5c291..8525f5c646c9d 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -1,6 +1,8 @@
This is a nonsense document, created for the sole purpose of
triggering libcxx premerge testing (so I can test metrics collection).
-
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
+
+Sample text.
+
>From 0bf2562b8313b4cbe82d4caf84b640bee13c686b Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Mon, 18 Aug 2025 11:15:49 -0700
Subject: [PATCH 06/15] Poke PR to kick off testing.
---
libcxx/docs/CarolinesTestDoc.txt | 2 --
1 file changed, 2 deletions(-)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index 8525f5c646c9d..36b255be72237 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -4,5 +4,3 @@ triggering libcxx premerge testing (so I can test metrics collection).
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
-Sample text.
-
>From 65b0c06039c3bdffe357e8610f1f9bb8ff12b3c3 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Mon, 18 Aug 2025 14:44:00 -0700
Subject: [PATCH 07/15] Poke PR.
---
libcxx/docs/CarolinesTestDoc.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index 36b255be72237..bf977fc515e2c 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -4,3 +4,4 @@ triggering libcxx premerge testing (so I can test metrics collection).
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
+Sample text goes here.
>From 09d22f42d876897e963229b3f63bf47110bbfba3 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Wed, 20 Aug 2025 07:58:24 -0700
Subject: [PATCH 08/15] Remove unnecessary name change.
---
.github/workflows/libcxx-build-and-test.yaml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 739e56e56f459..4b29c9296bf9d 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -35,7 +35,6 @@ concurrency:
jobs:
stage1:
- name: libc++ Stage1 Testing
if: github.repository_owner == 'llvm'
runs-on: llvm-premerge-libcxx-runners
continue-on-error: false
@@ -73,7 +72,6 @@ jobs:
**/CMakeOutput.log
**/crash_diagnostics/*
stage2:
- name: libc++ Stage2 Testing
if: github.repository_owner == 'llvm'
runs-on: llvm-premerge-libcxx-runners
needs: [ stage1 ]
@@ -119,7 +117,6 @@ jobs:
**/CMakeOutput.log
**/crash_diagnostics/*
stage3:
- name: libc++ Stage3 Testing
if: github.repository_owner == 'llvm'
needs: [ stage2 ]
continue-on-error: false
>From 5a930115b0fee56a3be85ee1f1de7f4ca57e7034 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Wed, 20 Aug 2025 10:57:02 -0700
Subject: [PATCH 09/15] Poke PR.
---
libcxx/docs/CarolinesTestDoc.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index bf977fc515e2c..e34bec83d623b 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -4,4 +4,4 @@ triggering libcxx premerge testing (so I can test metrics collection).
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
-Sample text goes here.
+Sample text goes here. And more sample text.
>From 1c405075ede64311181f5d12159b1fc4f16babad Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Wed, 20 Aug 2025 13:37:42 -0700
Subject: [PATCH 10/15] Poke PR.
---
libcxx/docs/CarolinesTestDoc.txt | 1 -
1 file changed, 1 deletion(-)
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
index e34bec83d623b..36b255be72237 100644
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ b/libcxx/docs/CarolinesTestDoc.txt
@@ -4,4 +4,3 @@ triggering libcxx premerge testing (so I can test metrics collection).
This is NOT intended to ever be reviewed or merged or committed. If it gets
accidentally committed, please remove it asap.
-Sample text goes here. And more sample text.
>From 2a32dfb66a298e1712d528ee87b7e238cbf7cf37 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Thu, 21 Aug 2025 08:36:44 -0700
Subject: [PATCH 11/15] Remove fake testing doc.
---
libcxx/docs/CarolinesTestDoc.txt | 6 ------
1 file changed, 6 deletions(-)
delete mode 100644 libcxx/docs/CarolinesTestDoc.txt
diff --git a/libcxx/docs/CarolinesTestDoc.txt b/libcxx/docs/CarolinesTestDoc.txt
deleted file mode 100644
index 36b255be72237..0000000000000
--- a/libcxx/docs/CarolinesTestDoc.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-This is a nonsense document, created for the sole purpose of
-triggering libcxx premerge testing (so I can test metrics collection).
-
-This is NOT intended to ever be reviewed or merged or committed. If it gets
-accidentally committed, please remove it asap.
-
>From c8c42b7697a17e2a374e27808397ea0a5bb8d7c6 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Thu, 21 Aug 2025 08:39:37 -0700
Subject: [PATCH 12/15] Add fake test document, for poking PR testing.
---
libcxx/docs/CarolineTest.txt | 4 ++++
1 file changed, 4 insertions(+)
create mode 100644 libcxx/docs/CarolineTest.txt
diff --git a/libcxx/docs/CarolineTest.txt b/libcxx/docs/CarolineTest.txt
new file mode 100644
index 0000000000000..6eaa0fcd3ca11
--- /dev/null
+++ b/libcxx/docs/CarolineTest.txt
@@ -0,0 +1,4 @@
+This is another test document.
+
+Please delete this document before merging any PRs; this document is NOT
+meant to be merged upstream.
>From 61ac17999b0f2358d2a1ee30e2bcf030938717e3 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Thu, 21 Aug 2025 08:54:49 -0700
Subject: [PATCH 13/15] Remove fake document (used for poking test runs).
Finish updating metrics.py to collect libc++ testing metrics.
---
.ci/metrics/metrics.py | 178 ++++++++++++++++++++++++++++++++---
libcxx/docs/CarolineTest.txt | 4 -
2 files changed, 167 insertions(+), 15 deletions(-)
delete mode 100644 libcxx/docs/CarolineTest.txt
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index 07f958f06f729..f3b7aab2eb53b 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -30,7 +30,7 @@
# This metric name is also used as a key in the job->name map.
GITHUB_WORKFLOW_TO_TRACK = {
"CI Checks": "github_llvm_premerge_checks",
- "Build and Test libc++": "github_libc++_premerge_checks",
+ "Build and Test libc++": "github_libcxx_premerge_checks",
}
# Lists the Github jobs to track for a given workflow. The key is the stable
@@ -42,10 +42,10 @@
"Build and Test Linux": "premerge_linux",
"Build and Test Windows": "premerge_windows",
},
- "github_libc++_premerge_checks": {
- "libc++ Stage1 Testing": "premerge_libcxx_stage1",
- "libc++ Stage2 Testing": "premerge_libcxx_stage2",
- "libc++ Stage3 Testing": "premerge_libcxx_stage3",
+ "github_libcxx_premerge_checks": {
+ "stage1": "premerge_libcxx_stage1",
+ "stage2": "premerge_libcxx_stage2",
+ "stage3": "premerge_libcxx_stage3",
},
}
@@ -70,13 +70,14 @@
# by trial and error).
GRAFANA_METRIC_MAX_AGE_MN = 120
-
@dataclass
class JobMetrics:
job_name: str
queue_time: int
run_time: int
status: int
+ created_at_ns: int
+ started_at_ns: int
completed_at_ns: int
workflow_id: int
workflow_name: str
@@ -89,6 +90,139 @@ class GaugeMetric:
time_ns: int
+ at dataclass
+class AggregateMetric:
+ aggregate_name: str
+ aggregate_queue_time: int
+ aggregate_run_time: int
+ aggregate_status: int
+ workflow_id: int
+
+
+def create_and_append_libcxx_aggregates(
+ workflow_metrics: list[JobMetrics]) -> list[JobMetrics,AggregateMetric]:
+ """
+ Find libc++ JobMetric entries and create aggregate metrics for them.
+
+ Sort the libc++ JobMetric entries by workflow id, and for each workflow
+ id group them by stages. Create an aggreate metric for each stage for each
+ unique workflow id. Append each aggregate metric to the workflow_metrics
+ list.
+
+ How aggreates are computed:
+ queue time: Time from when first job in group is created until last job in
+ group has started.
+ run time: Time from when first job in group starts running until last job
+ in group finishes running.
+ status: logical 'or' of all the job statuses in the group.
+ """
+ # Separate the jobs by workflow_id. Only look at JobMetrics entries.
+ aggregate_data = dict()
+ for job in workflow_metrics:
+ # Only want to look at JobMetrics
+ if not isinstance(job, JobMetrics):
+ continue
+ # Only want libc++ jobs.
+ if job.workflow_name != "Build and Test libc++":
+ continue
+ if job.workflow_id not in aggregate_data.keys():
+ aggregate_data[job.workflow_id] = [ job ]
+ else:
+ aggregate_data[job.workflow_id].append(job)
+
+ # Go through each aggregate_data list (workflow id) and find all the
+ # needed data
+ for ag_workflow_id in aggregate_data:
+ job_list = aggregate_data[ag_workflow_id]
+ stage1_jobs = list()
+ stage2_jobs = list()
+ stage3_jobs = list()
+ # sort jobs into stage1, stage2, & stage3.
+ for job in job_list:
+ if job.job_name.find('stage1') > 0:
+ stage1_jobs.append(job)
+ elif job.job_name.find('stage2') > 0:
+ stage2_jobs.append(job)
+ elif job.job_name.find('stage3') > 0:
+ stage3_jobs.append(job)
+
+ for job_list in [ stage1_jobs, stage2_jobs, stage3_jobs]:
+ if len(job_list) < 1:
+ # No jobs in that stage this time around.
+ continue
+
+ # Get the aggregate name.
+ ag_name = "github_libcxx_premerge_checks_"
+ if job_list[0].job_name.find('stage1') > 0:
+ ag_name = ag_name + "stage1_aggregate"
+ elif job_list[0].job_name.find('stage2') > 0:
+ ag_name = ag_name + "stage2_aggregate"
+ elif job_list[0].job_name.find('stage3') > 0:
+ ag_name = ag_name + "stage3_aggregate"
+ else:
+ ag_name = ag_name + "unknown_aggregate"
+
+ # Initialize the rest of the aggregate values
+ earliest_create = job_list[0].created_at_ns
+ earliest_start = job_list[0].started_at_ns
+ earliest_complete = job_list[0].completed_at_ns
+ latest_start = job_list[0].started_at_ns
+ latest_complete = job_list[0].completed_at_ns
+ ag_status = job_list[0].status
+
+ # Go through rest of jobs for this workflow id, updating stats
+ for job in job_list[1:]:
+ # Update the status
+ ag_status = ag_status or job.status
+ # Get the earliest & latest times
+ if job.created_at_ns < earliest_create:
+ earliest_create = job.created_at_ns
+ if job.completed_at_ns < earliest_complete:
+ earliest_complete = job.completed_at_ns
+ if job.started_at_ns > latest_start:
+ latest_start = job.started_at_ns
+ if job.started_at_ns < earliest_start:
+ earliest_start = job.started_at_ns
+ if job.completed_at_ns > latest_complete:
+ latest_complete = job.completed_at_ns
+
+ # Compute aggregate run time (in seconds, not ns)
+ ag_run_time = (latest_complete - earliest_start) / 1000000000
+ # Compute aggregate queue time (in seconds, not ns)
+ ag_queue_time = (latest_start - earliest_create) / 1000000000
+ # Append the aggregate metrics to the workflow metrics list.
+ workflow_metrics.append(
+ AggregateMetric(
+ ag_name, ag_queue_time, ag_run_time, ag_status,
+ ag_workflow_id
+ )
+ )
+ return
+
+def clean_up_libcxx_job_name(old_name: str) -> str:
+ """
+ Convert libcxx job names to generically legal strings.
+
+ Take a name like 'stage1 (generic-cxx03, clang-22, clang++-22)'
+ and convert it to 'stage1_generic_cxx03__clang_22__clangxx_22'.
+ (Remove parentheses; replace commas, hyphens and spaces with
+ underscores; replace '+' with 'x'.
+ """
+ # Names should have exactly one set of parentheses, so break on that. If
+ # they don't have any parentheses, then don't update them at all.
+ if old_name.find('(') == -1:
+ return old_name
+ stage, remainder = old_name.split('(')
+ stage = stage.strip()
+ if remainder[-1] == ')':
+ remainder = remainder[:-1]
+ remainder = remainder.replace('-', '_')
+ remainder = remainder.replace(',', '_')
+ remainder = remainder.replace(' ', '_')
+ remainder = remainder.replace('+', 'x')
+ new_name = stage + '_' + remainder
+ return new_name
+
def github_get_metrics(
github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
) -> tuple[list[JobMetrics], int]:
@@ -151,9 +285,14 @@ def github_get_metrics(
break
# This workflow is not interesting to us.
- if task.name not in GITHUB_WORKFLOW_TO_TRACK:
+ if (task.name not in GITHUB_WORKFLOW_TO_TRACK
+ and task.name != "Build and Test libc++"):
continue
+ libcxx_testing = False
+ if task.name == "Build and Test libc++":
+ libcxx_testing = True
+
if task.status == "completed":
workflow_seen_as_completed.add(task.id)
@@ -163,11 +302,20 @@ def github_get_metrics(
name_prefix = GITHUB_WORKFLOW_TO_TRACK[task.name]
for job in task.jobs():
+ if libcxx_testing:
+ # We're not running macos or windows libc++ tests on our
+ # infrastructure.
+ if (job.name.find("macos") != -1 or
+ job.name.find("windows") != -1):
+ continue
# This job is not interesting to us.
- if job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
+ elif job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
continue
- name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
+ if libcxx_testing:
+ name_suffix = clean_up_libcxx_job_name(job.name)
+ else:
+ name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
metric_name = name_prefix + "_" + name_suffix
if task.status != "completed":
@@ -216,8 +364,10 @@ def github_get_metrics(
continue
logging.info(f"Adding a job metric for job {job.id} in workflow {task.id}")
- # The timestamp associated with the event is expected by Grafana to be
- # in nanoseconds.
+ # The timestamp associated with the event is expected by Grafana to
+ # be in nanoseconds.
+ created_at_ns = int(created_at.timestamp()) * 10**9
+ started_at_ns = int(started_at.timestamp()) * 10**9
completed_at_ns = int(completed_at.timestamp()) * 10**9
workflow_metrics.append(
JobMetrics(
@@ -225,12 +375,18 @@ def github_get_metrics(
queue_time.seconds,
run_time.seconds,
job_result,
+ created_at_ns,
+ started_at_ns,
completed_at_ns,
task.id,
task.name,
)
)
+ # Finished collecting the JobMetrics for all jobs; now create the
+ # aggregates for any libc++ jobs.
+ create_and_append_libcxx_aggregates(workflow_metrics)
+
for name, value in queued_count.items():
workflow_metrics.append(
GaugeMetric(f"workflow_queue_size_{name}", value, time.time_ns())
diff --git a/libcxx/docs/CarolineTest.txt b/libcxx/docs/CarolineTest.txt
deleted file mode 100644
index 6eaa0fcd3ca11..0000000000000
--- a/libcxx/docs/CarolineTest.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-This is another test document.
-
-Please delete this document before merging any PRs; this document is NOT
-meant to be merged upstream.
>From 450117da40e0fd169831fe68633fa42bbb15e369 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Thu, 21 Aug 2025 15:00:28 -0700
Subject: [PATCH 14/15] Address reviewer comments,
---
.ci/metrics/metrics.py | 34 ++++++++++++++++++++++++++--------
1 file changed, 26 insertions(+), 8 deletions(-)
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index f3b7aab2eb53b..7d47c575df038 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -96,13 +96,21 @@ class AggregateMetric:
aggregate_queue_time: int
aggregate_run_time: int
aggregate_status: int
+ completed_at_ns: int
workflow_id: int
def create_and_append_libcxx_aggregates(
workflow_metrics: list[JobMetrics]) -> list[JobMetrics,AggregateMetric]:
- """
- Find libc++ JobMetric entries and create aggregate metrics for them.
+ """Find libc++ JobMetric entries and create aggregate metrics for them.
+
+ Args:
+ workflow_metrics: A list of JobMetrics entries collected so far.
+
+ Returns:
+ Returns a list of JobMetrics and AggregateMetric entries. It should
+ be the input list with the newly create AggregateMetric entries
+ appended to it.
Sort the libc++ JobMetric entries by workflow id, and for each workflow
id group them by stages. Create an aggreate metric for each stage for each
@@ -194,19 +202,25 @@ def create_and_append_libcxx_aggregates(
workflow_metrics.append(
AggregateMetric(
ag_name, ag_queue_time, ag_run_time, ag_status,
- ag_workflow_id
+ last_complete, ag_workflow_id
)
)
return
def clean_up_libcxx_job_name(old_name: str) -> str:
- """
- Convert libcxx job names to generically legal strings.
+ """Convert libcxx job names to generically legal strings.
+
+ Args:
+ old_name: A string with the full name of the libc++ test that was run.
+
+ Returns:
+ Returns the input string with characters that might not be acceptable
+ in some indentifier strings replaced with safer characters.
Take a name like 'stage1 (generic-cxx03, clang-22, clang++-22)'
and convert it to 'stage1_generic_cxx03__clang_22__clangxx_22'.
(Remove parentheses; replace commas, hyphens and spaces with
- underscores; replace '+' with 'x'.
+ underscores; replace '+' with 'x'.)
"""
# Names should have exactly one set of parentheses, so break on that. If
# they don't have any parentheses, then don't update them at all.
@@ -285,8 +299,7 @@ def github_get_metrics(
break
# This workflow is not interesting to us.
- if (task.name not in GITHUB_WORKFLOW_TO_TRACK
- and task.name != "Build and Test libc++"):
+ if (task.name not in GITHUB_WORKFLOW_TO_TRACK):
continue
libcxx_testing = False
@@ -442,6 +455,11 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
metrics_batch.append(
f"{name} queue_time={workflow_metric.queue_time},run_time={workflow_metric.run_time},status={workflow_metric.status} {workflow_metric.completed_at_ns}"
)
+ elif isinstance(workflow_metric, AggregateMetric):
+ name = workflow_metric.aggregate_name.lower().replace(" ", "_")
+ metrics_batch.append(
+ f"{name} queue_time={workflow_metric.aggregate_queue_time},run_time={workflow_metric.aggregate_run_time},status={workflow_metric.aggregate_status} {workflow_metric.completed_at_ns}"
+ )
else:
raise ValueError(
f"Unsupported object type {type(workflow_metric)}: {str(workflow_metric)}"
>From e725993d93ae9b9036bf71688d846511f720a3e6 Mon Sep 17 00:00:00 2001
From: Caroline Tice <cmtice at google.com>
Date: Fri, 22 Aug 2025 10:35:17 -0700
Subject: [PATCH 15/15] Add tests for libc++ metrics Fix two tiny bugs in
metrics.py
---
.ci/metrics/metrics.py | 4 +-
.ci/metrics/metrics_test.py | 250 +++++++++++++++++++++++++++++++++++-
2 files changed, 251 insertions(+), 3 deletions(-)
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index 7d47c575df038..b2399f0fb23ad 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -181,7 +181,7 @@ def create_and_append_libcxx_aggregates(
# Go through rest of jobs for this workflow id, updating stats
for job in job_list[1:]:
# Update the status
- ag_status = ag_status or job.status
+ ag_status = ag_status and job.status
# Get the earliest & latest times
if job.created_at_ns < earliest_create:
earliest_create = job.created_at_ns
@@ -202,7 +202,7 @@ def create_and_append_libcxx_aggregates(
workflow_metrics.append(
AggregateMetric(
ag_name, ag_queue_time, ag_run_time, ag_status,
- last_complete, ag_workflow_id
+ latest_complete, ag_workflow_id
)
)
return
diff --git a/.ci/metrics/metrics_test.py b/.ci/metrics/metrics_test.py
index 259e55f817939..cd63089af5468 100644
--- a/.ci/metrics/metrics_test.py
+++ b/.ci/metrics/metrics_test.py
@@ -36,7 +36,8 @@ def test_upload_gauge_metric(self):
def test_upload_job_metric(self):
"""Test that we can upload a job metric correctly."""
test_metrics = [
- metrics.JobMetrics("test_job", 5, 10, 1, 1000, 7, "test_workflow")
+ metrics.JobMetrics("test_job", 5, 10, 1, 1000, 1000, 1000, 7,
+ "test_workflow")
]
return_value = requests.Response()
return_value.status_code = 204
@@ -49,6 +50,22 @@ def test_upload_job_metric(self):
"test_job queue_time=5,run_time=10,status=1 1000",
)
+ def test_upload_aggregate_metric(self):
+ """Test that we can upload an aggregate metric correctly."""
+ test_metrics = [
+ metrics.AggregateMetric('stage1_aggregate', 211, 1124, 1, 1200, 9)
+ ]
+ return_value = requests.Response()
+ return_value.status_code = 204
+ with unittest.mock.patch(
+ "requests.post", return_value=return_value
+ ) as post_mock:
+ metrics.upload_metrics(test_metrics, "test_userid", "test_aoi_key")
+ self.assertEqual(
+ post_mock.call_args.kwargs["data"],
+ "stage1_aggregate queue_time=211,run_time=1124,status=1 1200",
+ )
+
def test_upload_unknown_metric(self):
"""Test we report an error if we encounter an unknown metric type."""
@@ -70,6 +87,237 @@ def test_bad_response_code(self):
with unittest.mock.patch("requests.post", return_value=return_value) as _:
metrics.upload_metrics(test_metrics, "test_userid", "test_api_key")
+ def test_create_and_append_aggregate_metric_1_stage(self):
+ """Test the creation of a single AggregateMetric"""
+ test_metrics = [
+ metrics.JobMetrics("libcxx_stage1_test1", 8, 388, 1,
+ created_at_ns=1755697953000000000,
+ started_at_ns=1755697961000000000,
+ completed_at_ns=1755698349000000000,
+ workflow_id=3,
+ workflow_name="Build and Test libc++"),
+ metrics.JobMetrics("libcxx_stage1_test2", 107, 357, 1,
+ created_at_ns=1755697953000000000,
+ started_at_ns=1755698060000000000,
+ completed_at_ns=1755698417000000000,
+ workflow_id=3,
+ workflow_name="Build and Test libc++"),
+ metrics.JobMetrics("libcxx_stage1_test3", 8, 824, 1,
+ created_at_ns=1755697953000000000,
+ started_at_ns=1755697961000000000,
+ completed_at_ns=1755698785000000000,
+ workflow_id=3,
+ workflow_name="Build and Test libc++"),
+ ]
+ metrics.create_and_append_libcxx_aggregates(test_metrics)
+ self.assertEqual(len(test_metrics), 4)
+ self.assertTrue(isinstance(test_metrics[-1], metrics.AggregateMetric))
+ aggregate = test_metrics[-1]
+ self.assertEqual(aggregate.aggregate_name,
+ "github_libcxx_premerge_checks_stage1_aggregate")
+ self.assertEqual(aggregate.aggregate_queue_time, 107)
+ self.assertEqual(aggregate.aggregate_run_time, 824)
+ self.assertEqual(aggregate.aggregate_status, 1)
+ self.assertEqual(aggregate.completed_at_ns, 1755698785000000000)
+ self.assertEqual(aggregate.workflow_id, 3)
+
+ def test_create_and_append_aggregate_metric_multiple_workflow_ids(self):
+ """Test creation of AggregateMetric for same stage with diff workflow ids."""
+ test_metrics = [
+ metrics.JobMetrics("libcxx_stage1_test1", 8, 388, 1,
+ created_at_ns=1755697953000000000,
+ started_at_ns=1755697961000000000,
+ completed_at_ns=1755698349000000000,
+ workflow_id=3,
+ workflow_name="Build and Test libc++"),
+ metrics.JobMetrics("libcxx_stage1_test2", 107, 357, 0,
+ created_at_ns=1755697953000000000,
+ started_at_ns=1755698060000000000,
+ completed_at_ns=1755698417000000000,
+ workflow_id=3,
+ workflow_name="Build and Test libc++"),
+ metrics.JobMetrics("libcxx_stage1_test3", 8, 824, 1,
+ created_at_ns=1755697953000000000,
+ started_at_ns=1755697961000000000,
+ completed_at_ns=1755698785000000000,
+ workflow_id=25,
+ workflow_name="Build and Test libc++"),
+ ]
+ metrics.create_and_append_libcxx_aggregates(test_metrics)
+ self.assertEqual(len(test_metrics), 5)
+ self.assertTrue(isinstance(test_metrics[3], metrics.AggregateMetric))
+ self.assertTrue(isinstance(test_metrics[4], metrics.AggregateMetric))
+ aggregate = test_metrics[3]
+ self.assertEqual(aggregate.aggregate_name,
+ "github_libcxx_premerge_checks_stage1_aggregate")
+ self.assertEqual(aggregate.aggregate_queue_time, 107)
+ self.assertEqual(aggregate.aggregate_run_time, 456)
+ self.assertEqual(aggregate.aggregate_status, 0)
+ self.assertEqual(aggregate.completed_at_ns, 1755698417000000000)
+ self.assertEqual(aggregate.workflow_id, 3)
+
+ aggregate = test_metrics[4]
+ self.assertEqual(aggregate.aggregate_name,
+ "github_libcxx_premerge_checks_stage1_aggregate")
+ self.assertEqual(aggregate.aggregate_queue_time, 8)
+ self.assertEqual(aggregate.aggregate_run_time, 824)
+ self.assertEqual(aggregate.aggregate_status, 1)
+ self.assertEqual(aggregate.completed_at_ns, 1755698785000000000)
+ self.assertEqual(aggregate.workflow_id, 25)
+
+ def test_create_and_append_aggregate_metric_3_stages(self):
+ """Test the creation of AggregateMetric for each of 3 stages."""
+ test_metrics = [
+ metrics.JobMetrics('libcxx_stage1_test1', 124, 1454, 1,
+ created_at_ns=1755696929000000000,
+ started_at_ns=1755697053000000000,
+ completed_at_ns=1755698507000000000,
+ workflow_id=17,
+ workflow_name='Build and Test libc++'),
+ metrics.JobMetrics('libcxx_stage1_test2', 129,827, 1,
+ created_at_ns=1755696929000000000,
+ started_at_ns=1755697058000000000,
+ completed_at_ns=1755697885000000000,
+ workflow_id=17,
+ workflow_name='Build and Test libc++'),
+
+ metrics.JobMetrics('libcxx_stage2_test1', 6, 580, 1,
+ created_at_ns=1755698507000000000,
+ started_at_ns=1755698513000000000,
+ completed_at_ns=1755699093000000000,
+ workflow_id=17,
+ workflow_name='Build and Test libc++'),
+ metrics.JobMetrics('libcxx_stage2_test2', 7, 473, 1,
+ created_at_ns=1755698507000000000,
+ started_at_ns=1755698514000000000,
+ completed_at_ns=1755698987000000000,
+ workflow_id=17,
+ workflow_name='Build and Test libc++'),
+ metrics.JobMetrics('libcxx_stage2_test3', 7, 820, 1,
+ created_at_ns=1755698507000000000,
+ started_at_ns=1755698514000000000,
+ completed_at_ns=1755699334000000000,
+ workflow_id=17,
+ workflow_name='Build and Test libc++'),
+ metrics.JobMetrics('libcxx_stage3_test1', 7, 919, 1,
+ created_at_ns=1755709005000000000,
+ started_at_ns=1755709012000000000,
+ completed_at_ns=1755709931000000000,
+ workflow_id=17,
+ workflow_name='Build and Test libc++'),
+ metrics.JobMetrics('libcxx_stage3_test2', 141, 834, 1,
+ created_at_ns=1755709005000000000,
+ started_at_ns=1755709146000000000,
+ completed_at_ns=1755709980000000000,
+ workflow_id=17,
+ workflow_name='Build and Test libc++'),
+ metrics.JobMetrics('libcxx_stage3_test3', 131, 370, 1,
+ created_at_ns=1755709005000000000,
+ started_at_ns=1755709136000000000,
+ completed_at_ns=1755709514000000000,
+ workflow_id=17,
+ workflow_name='Build and Test libc++'),
+ ]
+ metrics.create_and_append_libcxx_aggregates(test_metrics)
+ self.assertEqual(len(test_metrics), 11)
+ self.assertTrue(isinstance(test_metrics[8], metrics.AggregateMetric))
+ self.assertTrue(isinstance(test_metrics[9], metrics.AggregateMetric))
+ self.assertTrue(isinstance(test_metrics[10], metrics.AggregateMetric))
+ aggregate = test_metrics[8]
+ self.assertEqual(aggregate.aggregate_name,
+ "github_libcxx_premerge_checks_stage1_aggregate")
+ self.assertEqual(aggregate.aggregate_queue_time, 129)
+ self.assertEqual(aggregate.aggregate_run_time, 1454)
+ self.assertEqual(aggregate.aggregate_status, 1)
+ self.assertEqual(aggregate.completed_at_ns, 1755698507000000000)
+ self.assertEqual(aggregate.workflow_id, 17)
+
+ aggregate = test_metrics[9]
+ self.assertEqual(aggregate.aggregate_name,
+ "github_libcxx_premerge_checks_stage2_aggregate")
+ self.assertEqual(aggregate.aggregate_queue_time, 7)
+ self.assertEqual(aggregate.aggregate_run_time, 821)
+ self.assertEqual(aggregate.aggregate_status, 1)
+ self.assertEqual(aggregate.completed_at_ns, 1755699334000000000)
+ self.assertEqual(aggregate.workflow_id, 17)
+
+ aggregate = test_metrics[10]
+ self.assertEqual(aggregate.aggregate_name,
+ "github_libcxx_premerge_checks_stage3_aggregate")
+ self.assertEqual(aggregate.aggregate_queue_time, 141)
+ self.assertEqual(aggregate.aggregate_run_time, 968)
+ self.assertEqual(aggregate.aggregate_status, 1)
+ self.assertEqual(aggregate.completed_at_ns, 1755709980000000000)
+ self.assertEqual(aggregate.workflow_id, 17)
+
+ def test_create_and_append_aggregate_metric_mixed_job_types(self):
+ """Test the creation of AggregateMetric with non-lib++ jobs thrown in."""
+ test_metrics = [
+ metrics.JobMetrics("ci_test1", 5, 10, 1, 1000, 1200, 1400, 5,
+ "premerge_test"),
+ metrics.JobMetrics("libcxx_stage1_test1", 8, 388, 1,
+ created_at_ns=1755697953000000000,
+ started_at_ns=1755697961000000000,
+ completed_at_ns=1755698349000000000,
+ workflow_id=3,
+ workflow_name="Build and Test libc++"),
+ metrics.JobMetrics("ci_test2", 3, 20, 1, 2000, 2200, 2400, 37,
+ "premerge_test"),
+ metrics.JobMetrics("libcxx_stage1_test2", 107, 357, 0,
+ created_at_ns=1755697953000000000,
+ started_at_ns=1755698060000000000,
+ completed_at_ns=1755698417000000000,
+ workflow_id=3,
+ workflow_name="Build and Test libc++"),
+ metrics.JobMetrics("ci_test3", 7, 35, 1, 3000, 3200, 3400, 85,
+ "premerge_test"),
+ ]
+ metrics.create_and_append_libcxx_aggregates(test_metrics)
+ self.assertEqual(len(test_metrics), 6)
+ self.assertTrue(isinstance(test_metrics[5], metrics.AggregateMetric))
+ aggregate = test_metrics[5]
+ self.assertEqual(aggregate.aggregate_name,
+ "github_libcxx_premerge_checks_stage1_aggregate")
+ self.assertEqual(aggregate.aggregate_queue_time, 107)
+ self.assertEqual(aggregate.aggregate_run_time, 456)
+ self.assertEqual(aggregate.aggregate_status, 0)
+ self.assertEqual(aggregate.completed_at_ns, 1755698417000000000)
+ self.assertEqual(aggregate.workflow_id, 3)
+
+ def test_create_and_append_aggregate_metric_no_libcxx_jobs(self):
+ """Test the creation of AggregateMetric with no libc++ jobs.
+
+ In this case, no AggregateMetric should be created, but no
+ errors or complaints should be raised.
+ """
+ test_metrics = [
+ metrics.JobMetrics("ci_test1", 5, 10, 1, 1000, 1200, 1400, 5,
+ "premerge_test"),
+ metrics.JobMetrics("ci_test2", 3, 20, 1, 2000, 2200, 2400, 37,
+ "premerge_test"),
+ metrics.JobMetrics("ci_test3", 7, 35, 1, 3000, 3200, 3400, 85,
+ "premerge_test"),
+ ]
+ metrics.create_and_append_libcxx_aggregates(test_metrics)
+ self.assertEqual(len(test_metrics), 3)
+
+ def test_clean_up_libcxx_job_name(self):
+ """Test that we correctly update (or not) libcxx job names."""
+ stage1_name = "stage1 (test1, C++-test2, my-c++-test-25)"
+ stage2_name = "stage2 (generic-cxx26, clang-21, clang++21)"
+ stage3_name = "stage3 (generic-cxx26, libcxx-next-runners, junk)"
+ bad_name = "this is a bad name"
+ out_name1 = metrics.clean_up_libcxx_job_name(stage1_name)
+ self.assertEqual(out_name1,
+ "stage1_test1__Cxx_test2__my_cxx_test_25")
+ out_name2 = metrics.clean_up_libcxx_job_name(stage2_name)
+ self.assertEqual(out_name2,
+ "stage2_generic_cxx26__clang_21__clangxx21")
+ out_name3 = metrics.clean_up_libcxx_job_name(stage3_name)
+ self.assertEqual(out_name3,
+ "stage3_generic_cxx26__libcxx_next_runners__junk")
+ out_name4 = metrics.clean_up_libcxx_job_name(bad_name)
+ self.assertEqual(out_name4, bad_name)
if __name__ == "__main__":
unittest.main()
More information about the libcxx-commits
mailing list