[libcxx-commits] [libcxx] [libc++] Add scripts to run benchmarks and submit to LNT on a regular basis (PR #180849)

[Nikolas Klauser via libcxx-commits]

================
@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import argparse
+import json
+import logging
+import os
+import pathlib
+import platform
+import subprocess
+import sys
+import tempfile
+
+
+def directory_path(string):
+    if os.path.isdir(string):
+        return pathlib.Path(string)
+    else:
+        raise NotADirectoryError(string)
+
+def gather_machine_information(args):
+    """
+    Gather the machine information to upload to LNT as part of the submission.
+    """
+    info = {}
+    if platform.system() == 'Darwin':
+        profiler_info = json.loads(subprocess.check_output(['system_profiler', 'SPHardwareDataType', 'SPSoftwareDataType', '-json']).decode())
+        info['hardware'] = profiler_info['SPHardwareDataType'][0]['chip_type']
+        info['os'] = profiler_info['SPSoftwareDataType'][0]['os_version']
+        info['sdk'] = subprocess.check_output(['xcrun', '--show-sdk-version']).decode().strip()
+
+    info['compiler'] = subprocess.check_output([args.compiler, '--version']).decode().strip().splitlines()[0]
+    info['test_suite_commit'] = subprocess.check_output(['git', '-C', args.git_repo, 'rev-parse', args.test_suite_commit]).decode().strip()
+
+    return info
+
+def gather_run_information(args):
+    """
+    Gather the run information to upload to LNT as part of the submission.
+    """
+    info = {}
+    info['commit_info'] = subprocess.check_output(['git', '-C', args.git_repo, 'show', args.benchmark_commit, '--no-patch']).decode()
+    info['git_sha'] = subprocess.check_output(['git', '-C', args.git_repo, 'rev-parse', args.benchmark_commit]).decode().strip()
+    return info
+
+def dict_to_params(d):
+    """
+    Return a list of 'key=value' strings from a dictionary.
+    """
+    res = []
+    for (k, v) in d.items():
+        res.append(f'{k}={v}')
+    return res
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        prog='run-benchmarks',
+        description='Benchmark libc++ at the given commit and submit to LNT.')
+    parser.add_argument('--benchmark-commit', type=str, required=True,
+        help='The SHA representing the version of the library to benchmark.')
+    parser.add_argument('--test-suite-commit', type=str, required=True,
+        help='The SHA representing the version of the test suite to use for benchmarking.')
+    parser.add_argument('--compiler', type=str, required=True,
+        help='Path to the compiler to use for testing.')
+    parser.add_argument('--machine', type=str, required=True,
+        help='The name of the machine for reporting LNT results.')
+    parser.add_argument('--test-suite', type=str, required=True,
+        help='The name of the test suite for reporting LNT results.')
+    parser.add_argument('--lnt-url', type=str, required=True,
+        help='The URL of the LNT instance to submit results to.')
+    parser.add_argument('--disable-microbenchmarks', action='store_true',
+        help="Do not run the microbenchmarks, only run SPEC (if possible).")
+    parser.add_argument('--spec-dir', type=pathlib.Path, required=False,
+        help='Optional path to a SPEC installation to use for benchmarking.')
+    parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
+        help='Optional path to the Git repository to use. By default, the current working directory is used.')
+    parser.add_argument('--dry-run', action='store_true',
+        help='Only print what would be executed.')
+    parser.add_argument('-v', '--verbose', action='store_true',
+        help='Print the output of all subcommands.')
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(level=logging.INFO)
+
+    do_spec = args.spec_dir is not None
+    do_micro = not args.disable_microbenchmarks
+    if not (do_spec or do_micro):
+        raise ValueError("You must run at least the microbenchmarks or SPEC")
+
+    def run(command, *posargs, enforce_success=True, **kwargs):
----------------
philnik777 wrote:

Can we define this outside `main`?

https://github.com/llvm/llvm-project/pull/180849