[llvm] [polly] [Buildbot][Polly] Move polly-x86_64-linux-test-suite build instructions into main repository (PR #166809)

[Jan Patrick Lehr via llvm-commits]

================
@@ -0,0 +1,471 @@
+import argparse
+import filecmp
+import os
+import pathlib
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import traceback
+import platform
+import multiprocessing
+from contextlib import contextmanager
+
+_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])')
+_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile("[ \t\n]")
+
+
+def _shquote_windows(txt):
+    """shlex.quote for Windows cmd.exe"""
+    txt = txt.replace("%", "%%")
+    quoted = re.sub(_SHQUOTE_WINDOWS_ESCAPEDCHARS, r"\\\1", txt)
+    if len(quoted) == len(txt) and not _SHQUOTE_WINDOWS_QUOTEDCHARS.search(txt):
+        return txt
+    else:
+        return '"' + quoted + '"'
+
+
+def shjoin(args):
+    """Convert a list of shell arguments to an appropriately quoted string."""
+    if os.name in set(("nt", "os2", "ce")):
+        return " ".join(map(_shquote_windows, args))
+    else:
+        return shlex.join(args)
+
+
+def report(msg):
+    """Emit a message to the build log. Appears in red font. Lines surrounded by @@@ may be interpreted as meta-instructions."""
+    print(msg, file=sys.stderr, flush=True)
+
+
+def report_prog_version(name, cmd):
+    try:
+        p = subprocess.run(cmd, check=True, capture_output=True, text=True)
+        outlines = p.stdout.strip().splitlines()
+        report_list(name, outlines[0])
+    except BaseException:
+        pass
+
+
+def report_list(category, *items):
+    items = list(items)
+    filtered = []
+
+    while items:
+        item = items.pop()
+        match item:
+            case tuple() | list():
+                items += item
+                continue
+            case None:
+                continue
+            case _:
+                item = str(item).strip()
+        if not item:
+            continue
+        if item in filtered:
+            continue
+        filtered.append(item)
+    category += ":"
+    report(f"{category:<9}{', '.join(reversed( filtered))}")
+
+
+def report_platform():
+    report_list(
+        "CPU",
+        platform.machine(),
+        platform.architecture()[0],
+        platform.processor(),
+        f"{multiprocessing.cpu_count()} native threads",
+    )
+    try:
+        releaseinfo = platform.freedesktop_os_release()
+    except BaseException:
+        releaseinfo = dict()
+    report_list(
+        "OS",
+        platform.system(),
+        platform.architecture()[1],
+        platform.platform(),
+        releaseinfo.get("PRETTY_NAME"),
+    )
+    report_list("Python", platform.python_implementation(), platform.python_version())
+
+    report_prog_version("CMake", ["cmake", "--version"])
+    report_prog_version("Ninja", ["ninja", "--version"])
+    report_prog_version("Sphinx", ["sphinx-build", "--version"])
+    report_prog_version("Doxygen", ["doxygen", "--version"])
+
+    report_prog_version("gcc", ["gcc", "--version"])
+    report_prog_version("ld", ["ld", "--version"])
+
+    report_prog_version("LLVM", ["llvm-config", "--version"])
+    report_prog_version("Clang", ["clang", "--version"])
+    report_prog_version("LLD", ["ld.lld", "--version"])
+
+
+def run_command(cmd, shell=False, **kwargs):
+    """Report which command is being run, then execute it using subprocess.check_call."""
+    report(f"Running: {cmd if shell else shjoin(cmd)}")
+    sys.stderr.flush()
+    subprocess.check_call(cmd, shell=shell, **kwargs)
+
+
+def _remove_readonly(func, path, _):
+    """Clear the readonly bit and reattempt the removal."""
+    os.chmod(path, os.stat.S_IWRITE)
+    func(path)
+
+
+def rmtree(path):
+    """Remove directory path and all its subdirectories. Includes a workaround
+    for Windows where shutil.rmtree errors on read-only files.
+
+    Taken from official Pythons docs
+    https://docs.python.org/3/library/shutil.html#rmtree-example
+    """
+    shutil.rmtree(path, onexc=_remove_readonly)
+
+
+def checkout(giturl, sourcepath):
+    """
+    Use git to checkout the remote repository giturl at local directory sourcepath.
+
+    If the repository already exists, clear all local changes and check out the latest main branch.
+    """
+    if not os.path.exists(sourcepath):
+        run_command(["git", "clone", giturl, sourcepath])
+
+    # Reset repository state no matter what there was before
+    run_command(["git", "-C", sourcepath, "stash", "--all"])
+    run_command(["git", "-C", sourcepath, "stash", "clear"])
+
+    # Fetch and checkout the newest
+    run_command(["git", "-C", sourcepath, "fetch", "origin"])
+    run_command(["git", "-C", sourcepath, "checkout", "origin/main", "--detach"])
+
+
+ at contextmanager
+def step(step_name, halt_on_fail=False):
+    """Report a new build step being started.
+
+    Use like this::
+        with step("greet-step"):
+            report("Hello World!")
+    """
+    # Barrier to separate stdio output for the the previous step
+    sys.stderr.flush()
+    sys.stdout.flush()
+
+    report(f"@@@BUILD_STEP {step_name}@@@")
+    if halt_on_fail:
+        report("@@@HALT_ON_FAILURE@@@")
+    try:
+        yield
+    except Exception as e:
+        if isinstance(e, subprocess.CalledProcessError):
+            report(f"{shjoin(e.cmd)} exited with return code {e.returncode}.")
+            report("@@@STEP_FAILURE@@@")
+        else:
+            traceback.print_exc()
+            report("@@@STEP_EXCEPTION@@@")
+        if halt_on_fail:
+            # Do not continue with the next steps, but allow except/finally blocks to execute
+            raise e
+
+
+class Worker:
+    """Helper class to keep context in a worker.run() environment"""
+
+    def __init__(self, args, clean, clobber, workdir, jobs, cachefile, llvmsrcroot):
+        self.args = args
+        self.clean = clean
+        self.clobber = clobber
+        self.workdir = workdir
+        self.jobs = jobs
+        self.cachefile = cachefile
+        self.llvmsrcroot = llvmsrcroot
+
+    def in_llvmsrc(self, path):
+        """Convert a path in the llvm-project source checkout to an absolute path"""
+        return os.path.join(self.llvmsrcroot, path)
+
+    def in_workdir(self, path):
+        """Convert a path in the workdir to an absolute path"""
+        return os.path.join(self.workdir, path)
+
+    def run_ninja(
+        self, targets: list = [], *, builddir, ccache_stats: bool = False, **kwargs
+    ):
+        """Run ninja in builddir. If self.jobs is set, automatically adds an -j option to set the number of parallel jobs.
+
+        Parameters
+        ----------
+        targets : list
+            List of build targets; build the default target 'all' if list is empty
+        builddir
+            Directory of the build.ninja file
+        ccache_stats : bool
+            If true, also emit ccache statistics when finishing the build
+        """
+        cmd = ["ninja"]
+        if builddir is not None:
+            cmd += ["-C", builddir]
+        cmd += targets
+        if self.jobs:
+            cmd.append(f"-j{self.jobs}")
+        if ccache_stats:
+            run_command(["ccache", "-z"])
+            try:
+                run_command(cmd, **kwargs)
+            finally:
+                # TODO: Pipe to stderr to separate from build log itself
+                run_command(["ccache", "-sv"])
+        else:
+            run_command(cmd, **kwargs)
+
+    @contextmanager
+    def step(self, step_name, halt_on_fail=False):
+        """Convenience wrapper for step()"""
+        with step(step_name, halt_on_fail=halt_on_fail) as s:
+            yield s
+
+    def report(self, msg):
+        """Convenience wrapper for report()"""
+        report(msg)
+
+    def run_command(self, *args, **kwargs):
+        """Convenience wrapper for run_command()"""
+        return run_command(*args, **kwargs)
+
+    def rmtree(self, *args, **kwargs):
+        """Convenience wrapper for rmtree()"""
+        return rmtree(*args, *kwargs)
+
+    def checkout(self, giturl, sourcepath):
+        """Convenience wrapper for checkout()"""
+        return checkout(giturl, sourcepath)
+
+
+def convert_bool(v):
+    """Convert input to bool type
+
+    Use to convert the value of bool environment variables. Specifically, the
+    buildbot master sets 'false' to build properties, which by default Python
+    would interpret as true-ish.
+    """
+    match v:
+        case None:
+            return False
+        case bool(b):
+            return b
+        case str(s):
+            return not s.strip().upper() in ["", "0", "N", "NO", "FALSE", "OFF"]
+        case _:
+            return bool(v)
+
+
+def relative_if_possible(path, relative_to):
+    """Like os.path.relpath, but does not fail if path is not a parent of relative_to; keeps the original path in that case"""
+    path = os.path.normpath(path)
+    if not os.path.isabs(path):
+        # Path is already relative (assumed to relative_to)
+        return path
+    try:
+        result = os.path.relpath(path, start=relative_to)
+        return result if result else path
+    except ValueError:
+        return path
+
+
+ at contextmanager
+def run(
+    scriptpath,
+    llvmsrcroot,
+    parser=None,
+    clobberpaths=[],
+    workerjobs=None,
+    always_clobber=False,
+):
+    """
+    Runs the boilerplate for a ScriptedBuilder buildbot. It is not necessary to
+    use this function (one can also all run_command() etc. directly), but allows
+    for some more flexibility and safety checks. Arguments passed to this
+    function represent the worker configuration.
+
+    We use the term 'clean' for resetting the worker to an empty state. This
+    involves deleting ${prefix}/llvm.src as well as ${prefix}/build.
+    The term 'clobber' means deleting build artifacts, but not already
+    downloaded git repositories. Build artifacts including build- and
+    install-directories, but not source directories. Changes in the llvm.src
+    directory will be reset before the next build anyway. Clobber is necessary
+    if the build instructions change. Otherwise, we try an incremental build.
----------------
jplehr wrote:

Should this default to always clobber? The instructions on the LLVM documentation "How to setup a buildbot" strongly advise against incremental builds and rely on ccache instead. Should the default behavior here reflect that?

https://github.com/llvm/llvm-project/pull/166809