[libcxx-commits] [libcxx] [libc++] Add scripts to run benchmarks and submit to LNT on a regular basis (PR #180849)

[Nikolas Klauser via libcxx-commits]

================
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+from typing import List, Set
+import argparse
+import json
+import logging
+import os
+import pathlib
+import subprocess
+import sys
+
+
+def directory_path(string):
+    if os.path.isdir(string):
+        return pathlib.Path(string)
+    else:
+        raise NotADirectoryError(string)
+
+def api(lnt_url: str, test_suite: str, endpoint: str):
+    url = f'{lnt_url}/api/db_default/v4/{test_suite}{endpoint}'
+    logging.debug(f'Querying {url}')
+    result = json.loads(subprocess.check_output(['curl', '-sS', url]).decode())
+    return result
+
+def get_benchmarked_commits(lnt_url: str, test_suite: str, machine: str) -> Set[str]:
+    """
+    Return the set of commits that have already been benchmarked on the given LNT
+    instance, test suite and machine.
+    """
+    result = api(lnt_url, test_suite, f'/machines/{machine}')
+    commits = set()
+    if 'runs' not in result: # there is no such machine
+        return set()
+    for run in result['runs']:
+        if 'git_sha' not in run:
+            raise ValueError(f'Found run without a git_sha field: are you using the right LNT test suite? {run}')
+        commits.add(run['git_sha'])
+    return commits
+
+def git_rev_list(git_repo: str, paths: List[str] = []) -> List[str]:
+    """
+    Return the list of all revisions in the given Git repository. Older commits are earlier in the list.
+    """
+    cli = ['git', '-C', git_repo, 'rev-list', 'origin', '--', *paths]
+    rev_list = subprocess.check_output(cli).decode().strip().splitlines()
+    return list(reversed(rev_list))
+
+def git_sorted_revlist(git_repo: str, commits: List[str]) -> List[str]:
+    """
+    Return the list of commits sorted by their chronological order (from oldest to newest) in the
+    provided Git repository. Items earlier in the list are older than items later in the list.
+    """
+    revlist_cmd = ['git', '-C', git_repo, 'rev-list', '--no-walk'] + list(commits)
+    revlist = subprocess.check_output(revlist_cmd, text=True).strip().splitlines()
+    return list(reversed(revlist))
+
+def get_all_libcxx_commits(git_repo: str) -> Set[str]:
+    """
+    Return the set of commits available to benchmark for libc++: this is the list of
+    commits that touch code in libc++ that we care to benchmark.
+    """
+    logging.debug(f'Fetching {git_repo}')
+    subprocess.check_call(['git', '-C', git_repo, 'fetch', '--quiet', 'origin'])
+    return set(git_rev_list(git_repo, paths=['libcxx/include', 'libcxx/src']))
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        prog='commit-watch',
+        description='Watch for libc++ commits to run benchmarks on, and run the specified command on them.')
+    parser.add_argument('--machine', type=str, required=True,
+        help='The name of the machine that we are producing results for on the LNT instance.')
+    parser.add_argument('--test-suite', type=str, required=True,
+        help='The name of the test suite that we are producing results for on the LNT instance.')
+    parser.add_argument('--lnt-url', type=str, required=True,
+        help='The URL of the LNT instance to use as the source of truth for finding already-benchmarked commits.')
+    parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
+        help='Optional path to the Git repository to use. By default, the current working directory is used.')
+    parser.add_argument('callback', nargs=argparse.REMAINDER,
+        help='The command to run on each commit that we determine to require benchmarking. Should be provided last and '
+             'separated from other arguments with a `--`.')
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(level=logging.INFO)
+
+    # Gather callback command
+    if not args.callback:
+        raise ValueError('A callback must be provided')
+    else:
+        if args.callback[0] != '--':
+            raise ValueError('For clarity, the callback must be separated from other options by --')
+        callback = args.callback[1:]
----------------
philnik777 wrote:

```suggestion
    elif args.callback[0] != '--':
        raise ValueError('For clarity, the callback must be separated from other options by --')
    callback = args.callback[1:]
```
I think that makes is clearer what's going on?

https://github.com/llvm/llvm-project/pull/180849