[zorg] r304258 - add generic annotated builder and clang-with-thin-lto-windows
Bob Haarman via llvm-commits
llvm-commits at lists.llvm.org
Tue May 30 17:10:44 PDT 2017
Author: inglorion
Date: Tue May 30 19:10:44 2017
New Revision: 304258
URL: http://llvm.org/viewvc/llvm-project?rev=304258&view=rev
Log:
add generic annotated builder and clang-with-thin-lto-windows
Summary:
Adds a generic annotated builder that can be used to easily add
multiple build configurations. As a first configuration, a new builder
that builds and tests llvm, clang, and lld with ThinLTO on Windows is
included.
Reviewers: gkistanova, rnk, zturner
Reviewed By: gkistanova, rnk
Subscribers: mehdi_amini
Differential Revision: https://reviews.llvm.org/D33148
Added:
zorg/trunk/zorg/buildbot/builders/AnnotatedBuilder.py
zorg/trunk/zorg/buildbot/builders/annotated/
zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py
zorg/trunk/zorg/buildbot/builders/annotated/clang-with-thin-lto-windows.py
zorg/trunk/zorg/buildbot/builders/annotated/util.py
Modified:
zorg/trunk/buildbot/osuosl/master/config/builders.py
zorg/trunk/buildbot/osuosl/master/config/slaves.py
Modified: zorg/trunk/buildbot/osuosl/master/config/builders.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/builders.py?rev=304258&r1=304257&r2=304258&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/builders.py (original)
+++ zorg/trunk/buildbot/osuosl/master/config/builders.py Tue May 30 19:10:44 2017
@@ -821,6 +821,13 @@ def _get_lld_builders():
'factory': ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=16, lto='thin'),
'category' : 'lld'},
+ {'name': "clang-with-thin-lto-windows",
+ 'slavenames' :["windows-lld-thinlto-1"],
+ 'builddir': "clang-with-thin-lto-windows",
+ 'factory': AnnotatedBuilder.getAnnotatedBuildFactory(
+ script="clang-with-thin-lto-windows.py"),
+ 'category' : 'lld'},
+
{'name' : "clang-lld-x86_64-2stage",
'slavenames' : ["am1i-slv1", "am1i-slv3"],
'builddir' : "clang-lld-x86_64-2stage",
Modified: zorg/trunk/buildbot/osuosl/master/config/slaves.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/slaves.py?rev=304258&r1=304257&r2=304258&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/slaves.py (original)
+++ zorg/trunk/buildbot/osuosl/master/config/slaves.py Tue May 30 19:10:44 2017
@@ -158,6 +158,8 @@ def get_build_slaves():
create_slave("windows-gcebot1", properties={'jobs': 8}, max_builds=1),
# Windows Server 2012 x86_64 32-core GCE instance
create_slave("windows-gcebot2", properties={'jobs': 32}, max_builds=1),
+ # Windows Server 2016 x86_64 16-core GCE instance
+ create_slave("windows-lld-thinlto-1", max_builds=1),
# Ubuntu x86-64, 51GiB System memory Intel(R) Xeon(R) CPU @ 2.60GHz
create_slave("lldb-build1-ubuntu-1404", properties={'jobs': 32, 'loadaverage':32}, max_builds=1),
Added: zorg/trunk/zorg/buildbot/builders/AnnotatedBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/AnnotatedBuilder.py?rev=304258&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/AnnotatedBuilder.py (added)
+++ zorg/trunk/zorg/buildbot/builders/AnnotatedBuilder.py Tue May 30 19:10:44 2017
@@ -0,0 +1,64 @@
+import os
+
+import buildbot
+import buildbot.process.factory
+from buildbot.process.properties import WithProperties
+from buildbot.steps.shell import SetProperty, ShellCommand
+from buildbot.steps.source import SVN
+from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand
+
+
+def getAnnotatedBuildFactory(
+ script,
+ clean=False,
+ env=None,
+ timeout=1200):
+ """
+ Returns a new build factory that uses AnnotatedCommand, which
+ allows the build to be run by version-controlled scripts that do
+ not require a buildmaster restart to update.
+ """
+
+ f = buildbot.process.factory.BuildFactory()
+
+ if clean:
+ f.addStep(SetProperty(property='clean', command='echo 1'))
+
+ # We normally use the clean property to indicate that we want a
+ # clean build, but AnnotatedCommand uses the clobber property
+ # instead. Therefore, set clobber if clean is set to a truthy
+ # value. This will cause AnnotatedCommand to set
+ # BUILDBOT_CLOBBER=1 in the environment, which is how we
+ # communicate to the script that we need a clean build.
+ f.addStep(SetProperty(
+ property='clobber',
+ command='echo 1',
+ doStepIf=lambda step: step.build.getProperty('clean', False)))
+
+ merged_env = {
+ 'TERM': 'dumb' # Be cautious and disable color output from all tools.
+ }
+ if env is not None:
+ # Overwrite pre-set items with the given ones, so user can set
+ # anything.
+ merged_env.update(env)
+
+ scripts_dir = "annotated"
+ f.addStep(SVN(name='update-annotate-scripts',
+ mode='update',
+ svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/'
+ 'zorg/buildbot/builders/annotated',
+ workdir=scripts_dir,
+ alwaysUseLatest=True))
+
+ # Explicitly use '/' as separator, because it works on *nix and Windows.
+ script_path = "../%s/%s" % (scripts_dir, script)
+ f.addStep(AnnotatedCommand(name="annotate",
+ description="annotate",
+ timeout=timeout,
+ haltOnFailure=True,
+ command=WithProperties(
+ "python %(script)s --jobs=%(jobs:-)s",
+ script=lambda _: script_path),
+ env=merged_env))
+ return f
Added: zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py?rev=304258&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py (added)
+++ zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py Tue May 30 19:10:44 2017
@@ -0,0 +1,352 @@
+from __future__ import print_function
+
+import util
+
+import argparse
+import errno
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+from os.path import join as pjoin
+
+def get_argument_parser(*args, **kwargs):
+ ap = argparse.ArgumentParser(*args, **kwargs)
+ ap.add_argument('--jobs', help='Number of concurrent jobs to run')
+ return ap
+
+
+class AnnotatedBuilder:
+
+ """
+ Builder implementation that can be used with Buildbot's AnnotatedCommand.
+ Usage:
+ builder = AnnotatedBuilder()
+ builder.run_steps()
+
+ See run_steps() for parameters that can be passed to alter the behavior.
+ """
+
+ def halt_on_failure(self):
+ util.report('@@@HALT_ON_FAILURE@@@')
+
+ def report_build_step(self, step):
+ util.report('@@@BUILD_STEP %s@@@' % (step,))
+
+ def report_step_exception(self):
+ util.report('@@@STEP_EXCEPTION@@@')
+
+ def build_and_check_stage(
+ self,
+ stage,
+ build_dir,
+ source_dir,
+ cmake_args,
+ check_targets=None,
+ clean=True,
+ jobs=None):
+ stage_name = 'stage %s' % (stage,)
+ if clean:
+ self.clean_build_dir(stage_name, build_dir)
+ self.cmake(stage_name, build_dir, source_dir, cmake_args=cmake_args)
+ self.build(stage_name, build_dir, jobs)
+ if check_targets is not None:
+ self.check(stage_name, build_dir, check_targets, jobs)
+
+ def build_and_check_stages(
+ self,
+ stages,
+ build_dir,
+ source_dir,
+ cmake_args,
+ extra_cmake_args,
+ c_compiler,
+ cxx_compiler,
+ linker,
+ check_stages,
+ check_targets,
+ stage1_extra_cmake_args,
+ jobs=None):
+ if jobs:
+ cmake_args = [ '-DLLVM_LIT_ARGS=-sv -j %s' % (jobs,) ] + cmake_args
+ for stage in range(1, stages + 1):
+ stage_build_dir = pjoin(build_dir, 'stage%s' % (stage,))
+ if stage == 1:
+ s_cmake_args = cmake_args + stage1_extra_cmake_args
+ stage_clean = str(
+ os.environ.get('BUILDBOT_CLOBBER', '')) != ''
+ else:
+ previous_stage_bin = pjoin(
+ build_dir, 'stage%s' % (stage - 1,), 'bin')
+ s_cmake_args = self.stage_cmake_args(
+ cmake_args,
+ extra_cmake_args,
+ c_compiler,
+ cxx_compiler,
+ linker,
+ previous_stage_bin)
+ stage_clean = True
+ if check_stages[stage - 1]:
+ stage_check_targets = check_targets
+ else:
+ stage_check_targets = None
+ self.build_and_check_stage(
+ stage,
+ stage_build_dir,
+ source_dir,
+ s_cmake_args,
+ stage_check_targets,
+ stage_clean,
+ jobs)
+
+ def build(self, stage_name, build_dir, jobs=None):
+ self.report_build_step('%s build' % (stage_name,))
+ self.halt_on_failure()
+ cmd = ['ninja']
+ if jobs:
+ cmd += ['-j', str(jobs)]
+ util.report_run_cmd(cmd, cwd=build_dir)
+
+ def check(self, stage_name, build_dir, check_targets, jobs=None):
+ self.report_build_step('%s check' % (stage_name,))
+ self.halt_on_failure()
+ cmd = ['ninja']
+ if jobs:
+ cmd += ['-j', str(jobs)]
+ cmd += check_targets
+ util.report_run_cmd(cmd, cwd=build_dir)
+
+ def clean_build_dir(self, stage_name, build_dir):
+ self.report_build_step('%s clean' % (stage_name,))
+ self.halt_on_failure()
+ try:
+ util.clean_dir(build_dir)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ self.report_step_exception()
+ raise
+
+ def cmake(
+ self,
+ stage_name,
+ build_dir,
+ source_dir,
+ cmake='cmake',
+ cmake_args=None):
+ self.report_build_step('%s cmake' % (stage_name,))
+ self.halt_on_failure()
+ cmd = [cmake]
+ if cmake_args is not None:
+ cmd += cmake_args
+ cmd += [source_dir]
+ util.mkdirp(build_dir)
+ util.report_run_cmd(cmd, cwd=build_dir)
+
+ def cmake_compiler_flags(
+ self,
+ c_compiler,
+ cxx_compiler,
+ linker=None,
+ path=None):
+ c_compiler_flag = '-DCMAKE_C_COMPILER=%s' % (
+ util.cmake_pjoin(path, c_compiler),)
+ cxx_compiler_flag = '-DCMAKE_CXX_COMPILER=%s' % (
+ util.cmake_pjoin(path, cxx_compiler),)
+ if linker is None:
+ linker_flag = ''
+ else:
+ linker_flag = '-DCMAKE_LINKER=%s' % (
+ util.cmake_pjoin(path, linker),)
+ if os.name == 'nt':
+ c_compiler_flag += '.exe'
+ cxx_compiler_flag += '.exe'
+ linker_flag += '.exe'
+ flags = [
+ c_compiler_flag,
+ cxx_compiler_flag,
+ ]
+ if linker is not None:
+ flags += [
+ linker_flag,
+ ]
+ return flags
+
+ def compiler_binaries(self, compiler):
+ """
+ Given a symbolic compiler name, return a tuple
+ (c_compiler, cxx_compiler) with the names of the binaries
+ to invoke.
+ """
+ if compiler == 'clang':
+ return ('clang', 'clang++')
+ elif compiler == 'clang-cl':
+ return ('clang-cl', 'clang-cl')
+ else:
+ raise ValueError('Unsupported compiler type: %s' % (compiler,))
+
+ def stage_cmake_args(
+ self,
+ cmake_args,
+ extra_cmake_args,
+ c_compiler,
+ cxx_compiler,
+ linker,
+ previous_stage_bin):
+ return (
+ cmake_args + [
+ '-DCMAKE_AR=%s' % (
+ util.cmake_pjoin(previous_stage_bin, 'llvm-ar'),),
+ '-DCMAKE_RANLIB=%s' % (
+ util.cmake_pjoin(previous_stage_bin, 'llvm-ranlib'),),
+ ] + self.cmake_compiler_flags(
+ c_compiler, cxx_compiler, linker, previous_stage_bin) +
+ extra_cmake_args)
+
+ def set_environment(self, env=None, vs_tools=None, arch=None):
+ self.report_build_step('set-environment')
+ try:
+ new_env = {
+ 'TERM': 'dumb',
+ }
+ if os.name == 'nt':
+ if vs_tools is None:
+ vs_tools = os.path.expandvars('%VS140COMNTOOLS%')
+ if arch is None:
+ arch = os.environ['PROCESSOR_ARCHITECTURE'].lower()
+ else:
+ arch = arch.lower()
+ vcvars_path = pjoin(
+ vs_tools, '..', '..', 'VC', 'vcvarsall.bat')
+ cmd = util.shquote_cmd([vcvars_path, arch]) + ' && set'
+ output = subprocess.check_output(cmd, shell=True)
+ for line in output.splitlines():
+ var, val = line.split('=', 1)
+ new_env[var] = val
+
+ if env is not None:
+ new_env.epdate(env)
+
+ for (var, val) in new_env.items():
+ os.environ[var] = val
+
+ for var in sorted(os.environ.keys()):
+ util.report('%s=%s' % (var, os.environ[var]))
+ except:
+ self.report_step_exception()
+ raise
+
+ def update_sources(self, source_dir, projects, revision=None, svn='svn'):
+ self.report_build_step('update-sources')
+ self.halt_on_failure()
+ try:
+ for (project, path, uri) in projects:
+ if path is None:
+ path = source_dir
+ elif not os.path.isabs(path):
+ path = pjoin(source_dir, path)
+ util.report(
+ "Updating %s at %s from %s" % (project, util.shquote(path), uri))
+ if revision is None:
+ revision_args = []
+ else:
+ revision_args = ['-r', revision]
+ if os.path.exists(pjoin(path, '.svn')):
+ cmd = [svn, 'up'] + revision_args
+ else:
+ util.mkdirp(path)
+ cmd = [svn, 'co'] + revision_args + [uri, '.']
+ util.report_run_cmd(cmd, cwd=path)
+ except:
+ self.report_step_exception()
+ raise
+
+ def run_steps(
+ self,
+ stages=1,
+ check_targets=None,
+ check_stages=None,
+ extra_cmake_args=None,
+ stage1_extra_cmake_args=None,
+ revision=None,
+ compiler='clang',
+ linker='ld.lld',
+ env=None,
+ jobs=None):
+ """
+ stages: number of stages to run (default: 1)
+ check_targets: targets to run during the check phase (default: ['check-all'])
+ check_stages: stages for which to run the check phase
+ (array of bool, default: all True)
+ extra_cmake_args: extra arguments to pass to cmake (default: [])
+ stage1_extra_cmake_args: extra arguments to pass to cmake for stage 1
+ (default: use extra_cmake_args)
+ revision: revision to check out (default: os.environ['BUILDBOT_REVISION'],
+ or, if that is unset, the latest revision)
+ compiler: compiler to use after stage 1
+ ('clang' or 'clang-cl'; default 'clang')
+ linker: linker to use after stage 1
+ (None (let cmake choose) or 'lld' (default))
+ env: environment overrides (map; default is no overrides)
+ jobs: number of jobs to run concurrently (default: determine automatically)
+ """
+
+ # Set defaults.
+ if check_targets is None:
+ check_targets = ['check-all']
+ if check_stages is None:
+ check_stages = [True] * stages
+ if extra_cmake_args is None:
+ extra_cmake_args = []
+ if revision is None:
+ revision = os.environ.get('BUILDBOT_REVISION')
+ if stage1_extra_cmake_args is None:
+ stage1_extra_cmake_args = extra_cmake_args
+
+ c_compiler, cxx_compiler = self.compiler_binaries(compiler)
+
+ self.set_environment(env)
+
+ # Update sources.
+ cwd = os.getcwd()
+ source_dir = pjoin(cwd, 'llvm.src')
+ build_dir = pjoin(cwd, 'build')
+ svn_uri_pattern = 'http://llvm.org/svn/llvm-project/%s/trunk'
+ projects = []
+ projects.append(('llvm', None, svn_uri_pattern % ('llvm',)))
+ projects.append(
+ ('clang', pjoin('tools', 'clang'), svn_uri_pattern % ('cfe',)))
+ cmake_args = ['-GNinja']
+ for p in ['lld']:
+ projects.append((p, pjoin('tools', p), svn_uri_pattern % (p,)))
+
+ self.update_sources(source_dir, projects, revision=revision)
+
+ # Build and check stages.
+ self.build_and_check_stages(
+ stages,
+ build_dir,
+ source_dir,
+ cmake_args,
+ extra_cmake_args,
+ c_compiler,
+ cxx_compiler,
+ linker,
+ check_stages,
+ check_targets,
+ stage1_extra_cmake_args,
+ jobs)
+
+ return 0
+
+
+def main(argv):
+ ap = get_argument_parser()
+ args = ap.parse_args(argv[1:])
+ builder = AnnotatedBuilder()
+ builder.run_steps(jobs=args.jobs)
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
Added: zorg/trunk/zorg/buildbot/builders/annotated/clang-with-thin-lto-windows.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/annotated/clang-with-thin-lto-windows.py?rev=304258&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/annotated/clang-with-thin-lto-windows.py (added)
+++ zorg/trunk/zorg/buildbot/builders/annotated/clang-with-thin-lto-windows.py Tue May 30 19:10:44 2017
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+
+import os
+import sys
+import annotated_builder
+
+
+def main(argv):
+ ap = annotated_builder.get_argument_parser()
+ args = ap.parse_args(argv[1:])
+
+ stages = 2
+ stage1_extra_cmake_args = [
+ '-DCMAKE_BUILD_TYPE=Release',
+ '-DLLVM_TARGETS_TO_BUILD=X86',
+ ]
+ extra_cmake_args = stage1_extra_cmake_args + [
+ '-DLLVM_ENABLE_LTO=thin',
+ '-DLLVM_USE_LINKER=lld',
+ ]
+ check_targets = ['check-llvm', 'check-clang', 'check-lld']
+ check_stages = [True] * stages
+ check_stages[0] = False
+ if os.name == 'nt':
+ compiler = 'clang-cl'
+ linker = 'lld-link'
+ else:
+ compiler = 'clang'
+ linker = 'ld.lld'
+
+ builder = annotated_builder.AnnotatedBuilder()
+ builder.run_steps(stages=stages,
+ extra_cmake_args=extra_cmake_args,
+ stage1_extra_cmake_args=stage1_extra_cmake_args,
+ check_targets=check_targets,
+ check_stages=check_stages,
+ compiler=compiler,
+ linker=linker,
+ jobs=args.jobs)
+
+
+if __name__ == '__main__':
+ sys.path.append(os.path.dirname(__file__))
+ sys.exit(main(sys.argv))
Added: zorg/trunk/zorg/buildbot/builders/annotated/util.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/annotated/util.py?rev=304258&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/annotated/util.py (added)
+++ zorg/trunk/zorg/buildbot/builders/annotated/util.py Tue May 30 19:10:44 2017
@@ -0,0 +1,111 @@
+from __future__ import print_function
+
+import errno
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+
+def clean_dir(path):
+ """
+ Remove directory path if it exists and create a new, empty directory
+ in its place.
+ """
+ try:
+ shutil.rmtree(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ mkdirp(path)
+
+
+def cmake_pjoin(*args):
+ """
+ Join paths like safe_pjoin, but replace backslashes with forward
+ slashes on platforms where they are path separators. This prevents
+ CMake from choking when trying to decode what it thinks are escape
+ sequences in filenames.
+ """
+ result = safe_pjoin(*args)
+ if os.sep == '\\':
+ return result.replace('\\', '/')
+ else:
+ return result
+
+
+def report(msg):
+ sys.stderr.write(msg + '\n')
+ sys.stderr.flush()
+
+
+def report_run_cmd(cmd, shell=False, *args, **kwargs):
+ """
+ Print a command, then executes it using subprocess.check_call.
+ """
+ report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),))
+ sys.stderr.flush()
+ subprocess.check_call(cmd, shell=shell, *args, **kwargs)
+
+
+def mkdirp(path):
+ """Create directory path if it does not already exist."""
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+
+def safe_pjoin(dirname, *args):
+ """
+ Join path components with os.path.join, skipping the first component
+ if it is None.
+ """
+ if dirname is None:
+ return os.path.join(*args)
+ else:
+ return os.path.join(dirname, *args)
+
+
+def _shquote_impl(txt, escaped_chars, quoted_chars):
+ quoted = re.sub(escaped_chars, r'\\\1', txt)
+ if len(quoted) == len(txt) and not quoted_chars.search(txt):
+ return txt
+ else:
+ return '"' + quoted + '"'
+
+
+_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])')
+_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]')
+
+
+def shquote_posix(txt):
+ """Return txt, appropriately quoted for POSIX shells."""
+ return _shquote_impl(
+ txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS)
+
+
+_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])')
+_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]')
+
+
+def shquote_windows(txt):
+ """Return txt, appropriately quoted for Windows's cmd.exe."""
+ return _shquote_impl(
+ txt.replace('%', '%%'),
+ _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS)
+
+
+def shquote(txt):
+ """Return txt, appropriately quoted for use in a shell command."""
+ if os.name in set(('nt', 'os2', 'ce')):
+ return shquote_windows(txt)
+ else:
+ return shquote_posix(txt)
+
+
+def shquote_cmd(cmd):
+ """Convert a list of shell arguments to an appropriately quoted string."""
+ return ' '.join(map(shquote, cmd))
More information about the llvm-commits
mailing list