[libcxx] [llvm] [libc++] Add tools for gathering historical benchmark data (PR #212775)

[Aiden Grossman via llvm-commits]

================
@@ -0,0 +1,483 @@
+#!/usr/bin/env python3
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+from typing import Dict, List, NamedTuple, Optional, Sequence, TextIO
+import argparse
+import json
+import logging
+import pathlib
+import re
+import subprocess
+import sys
+import tabulate
+import urllib.parse
+
+
+class WorkItem(NamedTuple):
+    """
+    One line of the plan as produced by `plan-benchmarks`.
+    """
+    commit: str
+    machine: str
+    samples_have: int
+    samples_want: int
+    reason: str = ''
+
+    @staticmethod
+    def parse(line: str) -> 'WorkItem':
+        """
+        Read one line of a plan, validating every field.
+
+        A plan is normally produced by `plan-benchmarks`, but it can also be
+        hand-written, so we are careful about validating the data here.
+        """
+        item = json.loads(line)
+        if not isinstance(item, dict):
+            raise ValueError(f'expected a JSON object, got {type(item).__name__}')
+
+        def field(name: str, types, required: bool = True, default=None):
+            if name not in item:
+                if required:
+                    raise ValueError(f'missing required field {name}')
+                return default
+            value = item[name]
+            if isinstance(value, bool) or not isinstance(value, types):
+                raise ValueError(f'{name} has the wrong type: {value!r}')
+            return value
+
+        commit = field('commit', str)
+        if not is_sha(commit):
+            raise ValueError(f'commit is not a full 40-character SHA: {commit!r}')
+        machine = field('machine', str)
+        if not machine:
+            raise ValueError('machine is empty')
+        samples_have = field('samples_have', int)
+        samples_want = field('samples_want', int)
+        if samples_have < 0:
+            raise ValueError(f'samples_have is negative: {samples_have}')
+        if samples_want <= samples_have:
+            raise ValueError(f'samples_want {samples_want} is not more than samples_have '
+                             f'{samples_have}, so there is nothing to request')
+        return WorkItem(commit=commit.lower(),
+                        machine=machine,
+                        samples_have=samples_have,
+                        samples_want=samples_want,
+                        reason=field('reason', str, required=False, default=''))
+
+    @property
+    def target(self) -> 'Target':
+        """The unit of work this item is about."""
+        return Target(self.commit, self.machine)
+
+    @property
+    def runs_to_request(self) -> int:
+        """How many more runs this item is asking for."""
+        return self.samples_want - self.samples_have
+
+
+class Target(NamedTuple):
+    """
+    A unit of work: one commit benchmarked on one machine.
+
+    Plans, workflow runs and retry budgets are all keyed by this pair.
+    """
+    commit: str
+    machine: str
+
+
+class WorkflowRun(NamedTuple):
----------------
boomanaiden154 wrote:

An alternative to manually using `gh` to query the API and maintaining these classes would be to use PyGithub which already has these sorts of classes and uses the API directly without requiring the CLI tool. It does mean taking on a python dependency though.

https://github.com/llvm/llvm-project/pull/212775