[libcxx] [llvm] [libc++] Add tools for gathering historical benchmark data (PR #212775)

[Aiden Grossman via llvm-commits]

================
@@ -0,0 +1,483 @@
+#!/usr/bin/env python3
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+from typing import Dict, List, NamedTuple, Optional, Sequence, TextIO
+import argparse
+import json
+import logging
+import pathlib
+import re
+import subprocess
+import sys
+import tabulate
+import urllib.parse
+
+
+class WorkItem(NamedTuple):
+    """
+    One line of the plan as produced by `plan-benchmarks`.
+    """
+    commit: str
+    machine: str
+    samples_have: int
+    samples_want: int
+    reason: str = ''
+
+    @staticmethod
+    def parse(line: str) -> 'WorkItem':
+        """
+        Read one line of a plan, validating every field.
+
+        A plan is normally produced by `plan-benchmarks`, but it can also be
+        hand-written, so we are careful about validating the data here.
+        """
+        item = json.loads(line)
+        if not isinstance(item, dict):
+            raise ValueError(f'expected a JSON object, got {type(item).__name__}')
+
+        def field(name: str, types, required: bool = True, default=None):
+            if name not in item:
+                if required:
+                    raise ValueError(f'missing required field {name}')
+                return default
+            value = item[name]
+            if isinstance(value, bool) or not isinstance(value, types):
+                raise ValueError(f'{name} has the wrong type: {value!r}')
+            return value
+
+        commit = field('commit', str)
+        if not is_sha(commit):
+            raise ValueError(f'commit is not a full 40-character SHA: {commit!r}')
+        machine = field('machine', str)
+        if not machine:
+            raise ValueError('machine is empty')
+        samples_have = field('samples_have', int)
+        samples_want = field('samples_want', int)
+        if samples_have < 0:
+            raise ValueError(f'samples_have is negative: {samples_have}')
+        if samples_want <= samples_have:
+            raise ValueError(f'samples_want {samples_want} is not more than samples_have '
+                             f'{samples_have}, so there is nothing to request')
+        return WorkItem(commit=commit.lower(),
+                        machine=machine,
+                        samples_have=samples_have,
+                        samples_want=samples_want,
+                        reason=field('reason', str, required=False, default=''))
+
+    @property
+    def target(self) -> 'Target':
+        """The unit of work this item is about."""
+        return Target(self.commit, self.machine)
+
+    @property
+    def runs_to_request(self) -> int:
+        """How many more runs this item is asking for."""
+        return self.samples_want - self.samples_have
+
+
+class Target(NamedTuple):
+    """
+    A unit of work: one commit benchmarked on one machine.
+
+    Plans, workflow runs and retry budgets are all keyed by this pair.
+    """
+    commit: str
+    machine: str
+
+
+class WorkflowRun(NamedTuple):
+    """A run of the benchmark workflow, as the Github API describes it."""
+    identifier: int
+    target: Target
+    finished: bool
+
+
+class Decision(NamedTuple):
+    """What to do with one work item, and why."""
+    item: WorkItem
+    action: str # 'dispatch', 'skip' or 'defer'
+    jobs: int   # how many runs to request now
+    reason: str
+
+
+class Runs(NamedTuple):
+    """What the workflow is currently doing, counted per target."""
+    in_flight: Dict[Target, int]
+    completed: Dict[Target, int]
+    ids: Dict[Target, List[int]]      # in-flight run ids, for the summary
+
+    def in_flight_on(self, machine: str) -> int:
+        """How many runs are in flight on a machine, whatever they are benchmarking."""
+        return sum(n for (target, n) in self.in_flight.items() if target.machine == machine)
+
+
+# The name the benchmark workflow gives its runs. What a run is benchmarking can
+# only be recovered from its name (the inputs a workflow was dispatched with are
+# not available from the API) so this format is a contract with the
+# libcxx-benchmark-commit.yml workflow.
+RUN_NAME_PREFIX = '[libc++] Run benchmark suite against '
+RUN_NAME = re.compile(re.escape(RUN_NAME_PREFIX) + r'([0-9a-fA-F]{40}) on (\S+)$')
+
+
+class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter,
+                    argparse.RawDescriptionHelpFormatter):
+    """Show defaults, but keep the paragraphs of the description intact."""
+
+def is_sha(string: str) -> bool:
+    return len(string) == 40 and all(c in '0123456789abcdef' for c in string.lower())
+
+def positive(string: str) -> int:
+    value = int(string)
----------------
boomanaiden154 wrote:

If we're going to the effort of a nice error message for negative integers, maybe we want a nice one for non-numerical entries too?

https://github.com/llvm/llvm-project/pull/212775