[libcxx] [llvm] [libc++] Add tools for gathering historical benchmark data (PR #212775)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 09:01:46 PDT 2026
================
@@ -0,0 +1,483 @@
+#!/usr/bin/env python3
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+from typing import Dict, List, NamedTuple, Optional, Sequence, TextIO
+import argparse
+import json
+import logging
+import pathlib
+import re
+import subprocess
+import sys
+import tabulate
+import urllib.parse
+
+
+class WorkItem(NamedTuple):
+ """
+ One line of the plan as produced by `plan-benchmarks`.
+ """
+ commit: str
+ machine: str
+ samples_have: int
+ samples_want: int
+ reason: str = ''
+
+ @staticmethod
+ def parse(line: str) -> 'WorkItem':
+ """
+ Read one line of a plan, validating every field.
+
+ A plan is normally produced by `plan-benchmarks`, but it can also be
+ hand-written, so we are careful about validating the data here.
+ """
+ item = json.loads(line)
+ if not isinstance(item, dict):
+ raise ValueError(f'expected a JSON object, got {type(item).__name__}')
+
+ def field(name: str, types, required: bool = True, default=None):
+ if name not in item:
+ if required:
+ raise ValueError(f'missing required field {name}')
+ return default
+ value = item[name]
+ if isinstance(value, bool) or not isinstance(value, types):
+ raise ValueError(f'{name} has the wrong type: {value!r}')
+ return value
+
+ commit = field('commit', str)
+ if not is_sha(commit):
+ raise ValueError(f'commit is not a full 40-character SHA: {commit!r}')
+ machine = field('machine', str)
+ if not machine:
+ raise ValueError('machine is empty')
+ samples_have = field('samples_have', int)
+ samples_want = field('samples_want', int)
+ if samples_have < 0:
+ raise ValueError(f'samples_have is negative: {samples_have}')
+ if samples_want <= samples_have:
+ raise ValueError(f'samples_want {samples_want} is not more than samples_have '
+ f'{samples_have}, so there is nothing to request')
+ return WorkItem(commit=commit.lower(),
+ machine=machine,
+ samples_have=samples_have,
+ samples_want=samples_want,
+ reason=field('reason', str, required=False, default=''))
+
+ @property
+ def target(self) -> 'Target':
+ """The unit of work this item is about."""
+ return Target(self.commit, self.machine)
+
+ @property
+ def runs_to_request(self) -> int:
+ """How many more runs this item is asking for."""
+ return self.samples_want - self.samples_have
+
+
+class Target(NamedTuple):
+ """
+ A unit of work: one commit benchmarked on one machine.
+
+ Plans, workflow runs and retry budgets are all keyed by this pair.
+ """
+ commit: str
+ machine: str
+
+
+class WorkflowRun(NamedTuple):
+ """A run of the benchmark workflow, as the Github API describes it."""
+ identifier: int
+ target: Target
+ finished: bool
+
+
+class Decision(NamedTuple):
+ """What to do with one work item, and why."""
+ item: WorkItem
+ action: str # 'dispatch', 'skip' or 'defer'
+ jobs: int # how many runs to request now
+ reason: str
+
+
+class Runs(NamedTuple):
+ """What the workflow is currently doing, counted per target."""
+ in_flight: Dict[Target, int]
+ completed: Dict[Target, int]
+ ids: Dict[Target, List[int]] # in-flight run ids, for the summary
+
+ def in_flight_on(self, machine: str) -> int:
+ """How many runs are in flight on a machine, whatever they are benchmarking."""
+ return sum(n for (target, n) in self.in_flight.items() if target.machine == machine)
+
+
+# The name the benchmark workflow gives its runs. What a run is benchmarking can
+# only be recovered from its name (the inputs a workflow was dispatched with are
+# not available from the API) so this format is a contract with the
+# libcxx-benchmark-commit.yml workflow.
+RUN_NAME_PREFIX = '[libc++] Run benchmark suite against '
+RUN_NAME = re.compile(re.escape(RUN_NAME_PREFIX) + r'([0-9a-fA-F]{40}) on (\S+)$')
+
+
+class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter,
+ argparse.RawDescriptionHelpFormatter):
+ """Show defaults, but keep the paragraphs of the description intact."""
+
+def is_sha(string: str) -> bool:
+ return len(string) == 40 and all(c in '0123456789abcdef' for c in string.lower())
+
+def positive(string: str) -> int:
+ value = int(string)
+ if value < 1:
+ raise argparse.ArgumentTypeError(f'expected a positive integer, got {string}')
+ return value
+
+def non_negative(string: str) -> int:
+ value = int(string)
+ if value < 0:
+ raise argparse.ArgumentTypeError(f'expected a non-negative integer, got {string}')
+ return value
+
+class GithubError(Exception):
+ pass
+
+
+def parse_plan(stream: TextIO) -> List[WorkItem]:
+ """Parse a plan into a list of work items, in the order they were written."""
+ items: List[WorkItem] = []
+ for (number, line) in enumerate(stream, start=1):
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ items.append(WorkItem.parse(line))
+ except ValueError as error:
+ raise ValueError(f'line {number} of the plan is not a valid work item: {error}')
+ return items
+
+
+def collapse(items: Sequence[WorkItem]) -> List[WorkItem]:
+ """
+ Keep one item per commit and machine, the one asking for the most runs.
+
+ A plan can be concatenated from multiple sources or handwritten, so we
+ de-duplicate commits against the same machine to avoid duplicate work.
+ """
+ best: Dict[Target, WorkItem] = {}
+ for item in items:
+ existing = best.get(item.target)
+ if existing is None or item.runs_to_request > existing.runs_to_request:
+ best[item.target] = item
+ # Preserve the order of the plan: it is the order the work should be done in.
+ seen = set()
+ ordered: List[WorkItem] = []
+ for item in items:
+ if item.target not in seen:
+ seen.add(item.target)
+ ordered.append(best[item.target])
+ return ordered
+
+
+def target_of_run(title: str) -> Optional[Target]:
+ """
+ Return what a workflow run is benchmarking, or None if it can't be told.
+
+ The name has to match the workflow's `run-name` exactly.
+ """
+ match = RUN_NAME.match(title.strip())
+ return Target(match.group(1).lower(), match.group(2)) if match else None
+
+
+def run_gh(args: Sequence[str], stdin: Optional[str] = None) -> str:
+ command = ['gh', *args]
+ logging.debug(f'$ {" ".join(command)}')
+ try:
+ result = subprocess.run(command, check=True, text=True, input=stdin,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ except FileNotFoundError:
+ raise GithubError('gh was not found. This tool needs the Github CLI to be installed.')
+ except subprocess.CalledProcessError as error:
+ raise GithubError(f'{" ".join(command)} failed with status {error.returncode}: '
+ f'{error.stderr.strip()}')
+ return result.stdout
+
+
+def workflow_runs(repo: str, workflow: str) -> List[WorkflowRun]:
+ """
+ Return every run of a workflow whose name says what it benchmarks.
+
+ The whole history is walked, however Github only retains runs for 90 days.
+ Runs are extracted without filtering on status.
+
+ Cancelled runs or runs named in any other format are ignored: they either
+ predate this tooling or don't represent expended capacity towards benchmarking
+ a target.
+ """
+ query = urllib.parse.urlencode({'per_page': 100})
----------------
boomanaiden154 wrote:
Depending upon how many workflows we run in the past 90 days, this might get somewhat expensive in terms of GitHub API usage. It will probably be fine though.
Although filtering generally introduces a lot of problems with the API returning incorrect results from my experience.
https://github.com/llvm/llvm-project/pull/212775
More information about the llvm-commits
mailing list