[llvm-commits] [zorg] r170357 - in /zorg/trunk/buildbot/llvmlab/master: buildbot.tac config/__init__.py config/builderconstruction.py config/builders.py config/local.cfg config/phase_config.py config/schedulers.py config/slaves.py config/status.py master.cfg
David Dean
david_dean at apple.com
Mon Dec 17 11:36:47 PST 2012
Author: ddean
Date: Mon Dec 17 13:36:47 2012
New Revision: 170357
URL: http://llvm.org/viewvc/llvm-project?rev=170357&view=rev
Log:
Bring up basic phased buildmaster.
Added:
zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py
zorg/trunk/buildbot/llvmlab/master/config/phase_config.py
Modified:
zorg/trunk/buildbot/llvmlab/master/buildbot.tac
zorg/trunk/buildbot/llvmlab/master/config/__init__.py
zorg/trunk/buildbot/llvmlab/master/config/builders.py
zorg/trunk/buildbot/llvmlab/master/config/local.cfg
zorg/trunk/buildbot/llvmlab/master/config/schedulers.py
zorg/trunk/buildbot/llvmlab/master/config/slaves.py
zorg/trunk/buildbot/llvmlab/master/config/status.py
zorg/trunk/buildbot/llvmlab/master/master.cfg
Modified: zorg/trunk/buildbot/llvmlab/master/buildbot.tac
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/buildbot.tac?rev=170357&r1=170356&r2=170357&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/buildbot.tac (original)
+++ zorg/trunk/buildbot/llvmlab/master/buildbot.tac Mon Dec 17 13:36:47 2012
@@ -5,8 +5,33 @@
import os
basedir = os.path.dirname(os.path.abspath(__file__))
-configfile = r'master.cfg'
+rotateLength = 10000000
+maxRotatedFiles = 10
+
+# if this is a relocatable tac file, get the directory containing the TAC
+if basedir == '.':
+ import os.path
+ basedir = os.path.abspath(os.path.dirname(__file__))
+# note: this line is matched against to check that this is a buildmaster
+# directory; do not edit it.
application = service.Application('buildmaster')
-BuildMaster(basedir, configfile).setServiceParent(application)
+
+try:
+ from twisted.python.logfile import LogFile
+ from twisted.python.log import ILogObserver, FileLogObserver
+ logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+ maxRotatedFiles=maxRotatedFiles)
+ application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+except ImportError:
+ # probably not yet twisted 8.2.0 and beyond, can't set log yet
+ pass
+
+configfile = r'master.cfg'
+
+m = BuildMaster(basedir, configfile)
+m.setServiceParent(application)
+m.log_rotation.rotateLength = rotateLength
+m.log_rotation.maxRotatedFiles = maxRotatedFiles
+
Modified: zorg/trunk/buildbot/llvmlab/master/config/__init__.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/__init__.py?rev=170357&r1=170356&r2=170357&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/__init__.py (original)
+++ zorg/trunk/buildbot/llvmlab/master/config/__init__.py Mon Dec 17 13:36:47 2012
@@ -1,9 +1,12 @@
-import builders
-import slaves
-import status
-
# Load local options.
import os
import ConfigParser
options = ConfigParser.RawConfigParser()
options.read(os.path.join(os.path.dirname(__file__), 'local.cfg'))
+
+import builderconstruction
+import builders
+import phase_config
+import schedulers
+import slaves
+import status
Added: zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py?rev=170357&view=auto
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py (added)
+++ zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py Mon Dec 17 13:36:47 2012
@@ -0,0 +1,237 @@
+from zorg.buildbot.builders.LNTBuilder import CreateLNTNightlyFactory
+from zorg.buildbot.Artifacts import rsync_user, master_name
+from zorg.buildbot.builders.ClangBuilder import getClangMSVCBuildFactory
+from zorg.buildbot.builders.ClangBuilder import phasedClang
+from zorg.buildbot.builders.LLDBBuilder import getLLDBxcodebuildFactory
+
+"""
+Helper module to handle automatically constructing builder objects from a
+builder name.
+
+We use a relatively unambigous mangling of builder names to ensure that any
+particular builder name is always descriptive of its job.
+"""
+
+# Builder Construction Dispatch
+
+__all__ = ['construct']
+def construct(name):
+ """
+ construct(name) -> builder
+
+ Given a builder name, demangle the name and construct the appropriate
+ builder for it.
+ """
+
+ # First, determine the 'kind' of build we are doing. Compiler builds are a
+ # common case, so we specialize their names -- other builds should have
+ # their type delimited by '_'.
+ if name.startswith('clang-') or name.startswith('llvm-gcc-'):
+ kind,subname = 'compile',name
+ elif name.startswith('apple-clang'):
+ kind,subname = 'buildit',name
+ else:
+ if '_' not in name:
+ raise ValueError, "invalid builder name: %r" % name
+ kind,subname = name.split('_', 1)
+
+ # Dispatch based on the kind of build.
+ ctor = builder_kinds.get(kind)
+ if ctor is None:
+ raise ValueError, "invalid builder name: %r" % name
+
+ # Construct the builder.
+ builder = ctor(subname)
+
+ # Validate that the ctor didn't attempt to define the name, build directory,
+ # slaves or category.
+ if 'name' in builder:
+ raise ValueError, "name should not be defined by builder ctors!"
+ if 'builddir' in builder:
+ raise ValueError, "builddir should not be defined by builder ctors!"
+ if 'slavename' in builder or 'slavenames' in builder:
+ raise ValueError, "slaves should not be defined by builder ctors!"
+ if 'category' in builder:
+ raise ValueError, "categories should not be defined by builder ctors!"
+
+ # The build directory is always based on the name.
+ try:
+ builder['name'] = name
+ except TypeError:
+ raise ValueError, "invalid builder name: %r" % name
+ builder['builddir'] = 'build.%s' % name
+
+ return builder
+
+def construct_compiler_builder_from_name(name):
+ # Compiler builds are named following:
+ # <compiler>-<host arch>-<host os>-[<build cc>-]<build style>.
+ # if <build cc> is unspecified, then the most recent validated build
+ # for this builder will be used as <build cc>
+ # This can be undesirable. e.g. when building a Debug+Asserts compiler
+ # for the time being, DA builds will bootstrap with the most recent
+ # validated RA build
+ # At the moment, if anything is specified as <build cc> the build factory
+ # will default to host's default compiler
+
+ # Hack around x86_64 problem for now, to avoid changing builder names yet.
+ #
+ # FIXME: Use a better mangling.
+ params = name.replace("x86_64", "x86$64").split('_')
+ if len(params) == 2:
+ name, branch = params
+ name = name.replace("x86$64", "x86_64")
+ params = name.replace('llvm-gcc', 'llvm$gcc').split('-')
+ params = [p.replace('llvm$gcc', 'llvm-gcc')
+ for p in params]
+ if len(params) == 4:
+ compiler,host_arch,host_os,build_style = params
+ build_cc = None
+ elif len(params) == 5:
+ compiler,host_arch,host_os,build_cc,build_style = params
+ else:
+ raise ValueError, "invalid builder name: %r" % name
+
+ # Check if this is an MSVC builder.
+ if host_os == 'xp':
+ if compiler == 'clang':
+ # FIXME: This isn't using the host arch, build cc or
+ if (host_arch != 'i386' or build_cc != 'msvc9' or
+ build_style != 'DA'):
+ raise ValueError, "invalid builder name: %r" % name
+ # FIXME: Shouldn't have to hard code jobs or cmake path here.
+ return { 'factory' : getClangMSVCBuildFactory(
+ cmake = r"c:\Program Files\CMake 2.8\bin\cmake",
+ jobs = 4) }
+ else:
+ raise NotImplementedError
+
+ target_triple = '%s-apple-%s' % (host_arch, host_os)
+ config_options = ['--build=%s' % target_triple,
+ '--host=%s' % target_triple]
+
+ if build_style == 'DA':
+ build_config = "Debug+Asserts"
+ config_options.extend(['--disable-optimized'])
+ config_options.extend(['--enable-assertions'])
+ elif build_style == 'RA':
+ build_config = "Release+Asserts"
+ config_options.extend(['--enable-optimized'])
+ config_options.extend(['--enable-assertions'])
+ elif build_style == 'R':
+ build_config = "Release"
+ config_options.extend(['--enable-optimized'])
+ config_options.extend(['--disable-assertions'])
+ else:
+ raise ValueError, "invalid build style: %r" % build_style
+
+ # Passing is_bootstrap==False will specify the stage 1 compiler as the
+ # latest validated apple-clang style compiler.
+
+ # build_cc must be set for a bootstrapped compiler
+ if compiler == 'clang':
+ return { 'factory' : phasedClang(config_options,
+ is_bootstrap = (build_cc is None)) }
+ elif compiler == 'llvm-gcc':
+ # Currently, llvm-gcc builders do their own two-stage build,
+ # they don't use any prebuilt artifacts.
+
+ # Set the gxxincludedir.
+ if host_os == 'darwin9':
+ gxxincludedir = "/usr/include/c++/4.0.0"
+ elif host_os == 'darwin11':
+ gxxincludedir = "/usr/include/c++/v1"
+ else:
+ gxxincludedir = "/usr/include/c++/4.2.1"
+
+ # Construct the GCC style build triple:
+ if host_arch == "i386" and host_os.startswith("darwin"):
+ triple = 'i686-apple-%s' % host_os
+ elif host_arch == "x86_64" and host_os.startswith("darwin"):
+ triple = 'x86_64-apple-%s' % host_os
+ else:
+ raise ValueError, "invalid builder name: %r" % name
+
+ dst = rsync_user + '@' + master_name + ':~/artifacts/' + name + '/'
+ return {
+ 'factory' : LLVMGCCBuilder.getLLVMGCCBuildFactory(
+ jobs = '%(jobs)s', triple = triple,
+ gxxincludedir = gxxincludedir,
+ stage1_config = build_config, stage2_config = build_config,
+ package_dst = dst) }
+ else:
+ raise NotImplementedError
+
+def construct_lnt_builder_from_name(name):
+ # LNT builds are named following:
+ # <compiler under test>_<arch>_<options>.
+ # and all options are prefixed by '-' and no '-' can appear in an option.
+
+ # Hack around x86_64 problem for now, to avoid changing builder names yet.
+ #
+ # FIXME: Use a better mangling.
+ params = name.replace("x86_64", "x86$64").split('_')
+ params = [p.replace("x86$64", "x86_64")
+ for p in params]
+ if len(params) == 2:
+ cc_under_test,lnt_options_string = params
+ else:
+ raise ValueError, "invalid builder name: %r" % name
+ cc_path = None
+ cxx_path = None
+ # We assume that '-' will never occur in an LNT option. Sounds risky, no?
+ split_options = lnt_options_string.split('-')
+ arch = split_options[0]
+ lnt_options = ['-'+s
+ for s in split_options[1:]]
+
+ # Create the LNT flags.
+ lnt_flags = []
+
+ # If this is llvm-gcc, don't expect it to honor -arch (that is actually the
+ # "driver driver"). Hard coded to support only i386 and x86_64 for now.
+ if name.startswith('llvm-gcc'):
+ if arch == 'x86_64':
+ lnt_flags.extend(["--cflag", "-m64"])
+ elif arch == 'i386':
+ lnt_flags.extend(["--cflag", "-m32"])
+ else:
+ raise ValueError, "invalid builder name: %r" % name
+ else:
+ lnt_flags.extend(["--arch", arch])
+ for option in lnt_options:
+ # FIXME: Reorganize this.
+ if option in ('-g','-flto'):
+ lnt_flags.extend(["--cflag", option])
+ else:
+ lnt_flags.extend(["--optimize-option", option])
+ lnt_flags.append("--small")
+
+ return { 'factory' : CreateLNTNightlyFactory(lnt_flags, cc_path,
+ cxx_path,
+ parallel = True,
+ jobs = "2"),
+ 'properties' : {'use_builder' : cc_under_test } }
+
+def construct_lldb_builder_from_name(name):
+ cc_under_test = name
+ params = name.split('-')
+ if len(params) == 4:
+ compiler, host_arch, host_os, kind = params
+ else:
+ raise ValueError, "invalid builder name: %r" % name
+ lldb_triple = '-'.join([host_arch,host_os])
+ return { 'factory': getLLDBxcodebuildFactory()}
+
+builder_kinds = {
+ 'compile' : construct_compiler_builder_from_name,
+ 'lnt' : construct_lnt_builder_from_name,
+ 'lldb' : construct_lldb_builder_from_name}
+
+# Testing.
+
+if __name__ == '__main__':
+ from phase_config import phases
+ for phase in phases:
+ for build in phase['builders']:
+ print construct(build)
Modified: zorg/trunk/buildbot/llvmlab/master/config/builders.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/builders.py?rev=170357&r1=170356&r2=170357&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/builders.py (original)
+++ zorg/trunk/buildbot/llvmlab/master/config/builders.py Mon Dec 17 13:36:47 2012
@@ -1,568 +1,40 @@
-import buildbot
-import buildbot.process.factory
-from buildbot.steps.shell import WithProperties
-from buildbot.steps.trigger import Trigger
-from buildbot.schedulers import basic, timed, triggerable
-from buildbot.steps import source
-
-def setProperty(f, new_property, new_value):
- f.addStep(buildbot.steps.shell.SetProperty(command=['echo', new_value],
- property=new_property,
- description=['set property', new_property]))
- return f
-
-# This serves as a generic method to pull different repositories.
-def pullsrc(f, repo_name, URL, dir, pull_mode='clobber', def_branch='trunk', UseLatest=False):
- f.addStep(source.SVN(name='pull.' + repo_name,
- mode=pull_mode,
- baseURL=URL,
- defaultBranch=def_branch, workdir=dir, always_purge=True,
- alwaysUseLatest=UseLatest))
- return f
-
-def pullClang(f):
- pullsrc(f, 'clang', 'http://llvm.org/svn/llvm-project/cfe/', 'llvm/tools/clang')
- return f
-
-def pullllvm(f):
- pullsrc(f, 'llvm', 'http://llvm.org/svn/llvm-project/llvm/', 'llvm')
- return f
-
-def pulltest_suite(f):
- pullsrc(f, 'llvm tests', 'http://llvm.org/svn/llvm-project/test-suite/', 'test-suite',
- 'clobber', 'trunk', 'True')
- return f
-
-def pullclang_tests(f):
- pullsrc(f, 'clang tests', 'http://llvm.org/svn/llvm-project/clang-tests/', 'test-suite')
- return f
-
-def pullboostrunner(f):
- pullsrc(f, 'boost.runner', 'http://svn.boost.org/svn/boost/%%BRANCH%%/tools/regression/src/',
- 'boost_runner', 'clobber', 'trunk', 'True')
- return f
-
-def getBuildDir(f):
- f.addStep(buildbot.steps.shell.SetProperty(name='get.build.dir',
- command=['pwd'],
- property='builddir',
- description='set build dir',
- workdir='.'))
- return f
-
-def GetCompilerArtifacts(f):
- # The 'revision' property is always defined, so it is safer to use.
- # That being said, it has not yet been set to a valid number for phase 1 builds.
- # You must use 'got_build' to determine which revision was actually pulled.
- if WithProperties('%(revision)s')=='None':
- src_file = WithProperties('buildmaster at llvmlab.local:~/artifacts/%(use_builder)s/' +
- 'clang-r%(got_revision)s-*.tar.gz')
- else:
- src_file = WithProperties('buildmaster at llvmlab.local:~/artifacts/%(use_builder)s/' +
- 'clang-r%(revision)s-*.tar.gz')
- slavedest=WithProperties('%(builddir)s/clang-host.tar.gz')
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='download.artifacts',
- command=['rsync', '-ave', 'ssh', src_file, slavedest],
- haltOnFailure=True,
- description=['download build artifacts'],
- workdir=WithProperties('%(builddir)s')))
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='unzip',
- command=['tar', '-zxvf','../clang-host.tar.gz'],
- haltOnFailure=True,
- description=['extract', 'clang-host'],
- workdir='clang-host'))
- return f
-
-def cleanCompilerDir(f):
- f.addStep(buildbot.steps.shell.ShellCommand(
- command=['rm', '-rfv', 'clang-install'],
- haltOnFailure=False,
- description=['rm dir', 'clang-install'],
- workdir=WithProperties('%(builddir)s')))
- f.addStep(buildbot.steps.shell.ShellCommand(
- command=['rm', '-rfv', 'clang-host'],
- haltOnFailure=False,
- description=['rm dir', 'clang-host'],
- workdir=WithProperties('%(builddir)s')))
- f.addStep(buildbot.steps.shell.ShellCommand(
- command=['sh', '-c', 'rm -rfv clang*.tar.gz'],
- haltOnFailure=False,
- description=['rm archives'],
- workdir=WithProperties('%(builddir)s')))
- f.addStep(buildbot.steps.shell.ShellCommand(
- command=['rm', '-rfv', WithProperties('%(compiler_built:-)s')],
- haltOnFailure=False,
- description=['rm dir', WithProperties('%(compiler_built:-)s')],
- workdir=WithProperties('%(builddir)s')))
- return f
-
-def uploadArtifacts(f):
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='tar.and.zip',
- command=['tar', 'czvf',
- WithProperties('../clang-r%(got_revision)s-b%(buildnumber)s.tar.gz'),
- './'],
- haltOnFailure=True,
- description=['tar', '&', 'zip'],
- workdir='clang-install'))
- archive_src = WithProperties('%(builddir)s/clang-r%(got_revision)s-b%(buildnumber)s.tar.gz')
- archive_dest = WithProperties('buildmaster at llvmlab.local:~/artifacts/%(buildername)s')
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='upload.artifacts',
- command=['rsync', '-ave', 'ssh', archive_src, archive_dest],
- haltOnFailure=True,
- description=['upload build artifacts'],
- workdir=WithProperties('%(builddir)s')))
- artifactsURL = WithProperties('http://smooshlab.apple.com/artifacts/%(buildername)s' +
- '/clang-r%(got_revision)s-b%(buildnumber)s.tar.gz')
- setProperty(f, 'artifactsURL', artifactsURL)
- return f
-
-def regressionTests(f):
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='run.llvm.tests',
- command=['make', '-j', WithProperties('%(jobs)s')],
- haltOnFailure=True,
- description=['llvm', 'tests'],
- env={'PATH':
- WithProperties('%(builddir)s/%(compiler_built)s/%(compiler_type)s/bin:${PATH}')},
- workdir=WithProperties('%(compiler_built)s/test')))
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='run.clang.tests',
- command=['make', '-j', WithProperties('%(jobs)s')],
- haltOnFailure=True,
- description=['clang', 'tests'],
- env={'PATH':
- WithProperties('%(builddir)s/%(compiler_built)s/%(compiler_type)s/bin:${PATH}')},
- workdir=WithProperties('%(compiler_built)s/tools/clang/test')))
- return f
-
-
-def createPhase1():
- # Create an instance of the Builder.
- f = buildbot.process.factory.BuildFactory()
- f = clangStage1(f)
- # Save artifacts of thids build for use by other builders.
- f = uploadArtifacts(f)
- f = regressionTests(f)
- return f
-
-
-def clangStage1(f, config_options=''):
- # Determine the build directory.
- f = getBuildDir(f)
- # Clean out the directories/archives used to ensure a clean build.
- f = cleanCompilerDir(f)
- # Pull sources.
- f = pullllvm(f)
- f = pullClang(f)
- # Configure to use built-in compiler.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='configure',
- command=[
- '../llvm/configure', '--enable-optimized', '--disable-bindings',
- '--with-llvmcc=clang', '--without-llvmgcc', '--without-llvmgxx',
- WithProperties('--prefix=/')],
- haltOnFailure=True,
- description=['configure'],
- workdir=WithProperties('%(compiler_built)s')))
- # Build clang.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='make',
- command=['make', '-j', WithProperties('%(jobs)s')],
- haltOnFailure=True,
- description=['make'],
- workdir=WithProperties('%(compiler_built)s')))
- # Use make inatall-clang to produce minimal archive for use by downstream builders.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='make.install-clang',
- command=['make', 'install-clang', '-j', WithProperties('%(jobs)s'),
- WithProperties('DESTDIR=%(builddir)s/clang-install')],
- haltOnFailure=True,
- description=['make install'],
- workdir=WithProperties('%(compiler_built)s')))
- return f
-
-def clangHost(config_options):
- # Create an instance of the Builder.
- f = buildbot.process.factory.BuildFactory()
- # Determine the build directory.
- f = getBuildDir(f)
- f = setProperty(f, 'use_path', WithProperties('%(builddir)s/clang-host/bin'))
- # Clean out the directories/archives used to ensure a clean build.
- f = cleanCompilerDir(f)
- # Pull sources.
- f = pullllvm(f)
- f = pullClang(f)
- # Download artifacts from phase 1 compiler build.
- f = GetCompilerArtifacts(f)
- if config_options == ():
- config_options = []
- else:
- config_options = list(config_options)
- # Configure to use artifacts from upstream builder.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='configure.with.host',
- command=[
- '../llvm/configure'] + config_options + ['--disable-bindings',
- '--with-llvmcc=clang', '--without-llvmgcc', '--without-llvmgxx',
- WithProperties('CC=%(use_path)s/clang'),
- WithProperties('CXX=%(use_path)s/clang++'),
- WithProperties('--prefix=/')],
- haltOnFailure=True,
- description=['configure'],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- workdir=WithProperties('%(compiler_built)s')))
- # Build clang using clang.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='stage.2.make',
- command=['make', '-j', WithProperties('%(jobs)s')],
- haltOnFailure=True,
- description=['make'],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- workdir=WithProperties('%(compiler_built)s')))
- # Use make inatall-clang to produce minimal archive for use by downstream builders.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='make.install-clang',
- command=['make', 'install-clang', '-j', WithProperties('%(jobs)s'),
- WithProperties('DESTDIR=%(builddir)s/clang-install')],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- haltOnFailure=True,
- description=['make install'],
- workdir=WithProperties('%(compiler_built)s')))
- # Save artifacts of thids build for use by other builders.
- f = uploadArtifacts(f)
- f = regressionTests(f)
- return f
-
-def getGatedFactory(buildphase, next):
- f = buildbot.process.factory.BuildFactory()
- f.addStep(Trigger(schedulerNames=[buildphase],
- waitForFinish=True,
- haltOnFailure=True,
- updateSourceStamp=True,
- set_properties={'revision': WithProperties('%(revision)s'),
- 'got_revision': WithProperties('%(revision)s')}))
- f.addStep(Trigger(schedulerNames=[next],
- waitForFinish=False,
- updateSourceStamp=True,
- set_properties={'revision': WithProperties('%(revision)s'),
- 'got_revision': WithProperties('%(revision)s')}))
- return f
-
-def PublishGoodBuild():
- f = buildbot.process.factory.BuildFactory()
- # TODO: Add steps to prepare a release and announce a good build.
- return f
-
-def Placeholder():
- f = buildbot.process.factory.BuildFactory()
- return f
-
-def makePhaseBuilder(bldname, trigger1, trigger2, bldslaves):
- return { 'name' : bldname,
- 'factory' : getGatedFactory(trigger1, trigger2),
- 'slavenames' : bldslaves ,
- 'category' : 'status'}
-
-def HostedClang(myname, compiler_type, use_compiler, slaves, *config_options):
- return { 'name' : myname,
- 'builddir' : 'build.'+myname,
- 'factory' : clangHost(config_options),
- 'slavenames' : slaves,
- 'category' : 'clang',
- 'properties' : {'compiler_type': compiler_type,
- 'use_builder': use_compiler,
- 'compiler_built': 'clang-build'}}
-
-def CreateNightly(options):
- f = buildbot.process.factory.BuildFactory()
- NightlyFactory(f, options)
- return f
-
-from zorg.buildbot.commands.NightlyTestCommand import NightlyTestCommand
-def NightlyFactory(f, options, clean=True, test=True, xfails=set()):
- # Determine the build directory.
- f = getBuildDir(f)
- f = setProperty(f, 'use_path', WithProperties('%(builddir)s/clang-host/bin'))
- # Clean out the directories/archives used to ensure a clean build.
- f = cleanCompilerDir(f)
- # Download compiler artifacts to be used for this build.
- f = GetCompilerArtifacts(f)
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='sanity.test',
- command=[WithProperties('%(use_path)s/clang'),
- '--version'],
- haltOnFailure=True,
- description=['sanity test'],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')}))
- # Pull source code.
- f = pulltest_suite(f)
- # Clean up.
- if clean:
- f.addStep(buildbot.steps.shell.ShellCommand(
- name="rm.test-suite",
- command=["rm", "-rfv", "test-suite-build"],
- haltOnFailure=True,
- description="rm test-suite build dir",
- workdir=WithProperties('%(builddir)s')))
- # Configure tests to use clang from upstream build.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='configure.tests',
- command=['../test-suite/configure',
- WithProperties('CC=%(use_path)s/clang'),
- WithProperties('CXX=%(use_path)s/clang++'),
- 'CFLAGS='+options,
- 'CXXFLAGS='+options,
- '--without-llvmsrc', '--without-llvmobj'],
- haltOnFailure=True,
- description=['configure tests'],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- workdir='test-suite-build'))
- # You need to make the tools target.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='make.tools',
- command=['make', 'tools', WithProperties('-j%(jobs)s')],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- haltOnFailure=True,
- description=['Make', 'tools'],
- workdir='test-suite-build'))
- # Build and test.
- f.addStep(NightlyTestCommand(
- name='run.fast.nightly.tests',
- command=['make', WithProperties('-j%(jobs)s'), 'ENABLE_PARALLEL_REPORT=1',
- 'DISABLE_CBE=1', 'DISABLE_JIT=1', 'TEST=simple', 'report'],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- haltOnFailure=True,
- description=['run', 'test-suite'],
- workdir='test-suite-build',
- logfiles={ 'report' : 'report.nightly.txt'},
- xfails=xfails))
- return f
-
-def Nightly(compiler, slaves, options=''):
- return { 'name' : 'nightly_'+ compiler + options,
- 'builddir' : 'build.nightly.'+ compiler + options,
- 'factory' : CreateNightly(options),
- 'slavenames' : slaves,
- 'category' : 'tests',
- 'properties' : {'use_builder': compiler}}
-
-def stage1Clang(compiler, compiler_type, slave):
- return { 'name' : compiler,
- 'builddir' : 'build.'+ compiler,
- 'factory' : createPhase1(),
- 'slavename' : slave,
- 'category' : 'clang',
- 'properties' : {'compiler_type': compiler_type,
- 'compiler_built': 'clang-build'}}
-
-def HostStage3Clang(config_options):
- # Create instance of Builder.
- f = buildbot.process.factory.BuildFactory()
- # Determine the build directory.
- f = getBuildDir(f)
- f = setProperty(f, 'use_path', WithProperties('%(builddir)s/clang-host/bin'))
- # Clean out the directories/archives used to ensure a clean build.
- f = cleanCompilerDir(f)
- # Pull sources.
- f = pullllvm(f)
- f = pullClang(f)
- # Download artifacts from upstream builder.
- f = GetCompilerArtifacts(f)
- # Configure to use compiler from upstream builder.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='configure.with.host',
- command=[
- '../llvm/configure', '--enable-optimized', '--disable-bindings',
- '--with-llvmcc=clang', '--without-llvmgcc', '--without-llvmgxx',
- config_options,
- WithProperties('CC=%(use_path)s/clang'),
- WithProperties('CXX=%(use_path)s/clang++'),
- WithProperties('--prefix=/')],
- haltOnFailure=True,
- description=['configure'],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- workdir=WithProperties('%(compiler_built)s')))
- # Build clang using clang.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='stage.3.make',
- command=['make', '-j', WithProperties('%(jobs)s')],
- haltOnFailure=True,
- description=['make'],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- workdir=WithProperties('%(compiler_built)s')))
- # Use make inatall-clang to produce minimal archive for use by downstream builders.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='make.install-clang',
- command=['make', 'install-clang', '-j', WithProperties('%(jobs)s'),
- WithProperties('DESTDIR=%(builddir)s/clang-install')],
- env={'PATH': WithProperties('%(use_path)s:${PATH}')},
- haltOnFailure=True,
- description=['make install'],
- workdir=WithProperties('%(compiler_built)s')))
- # Save artifacts of thids build for use by downstream builders.
- f = uploadArtifacts(f)
- f = regressionTests(f)
- NightlyFactory(f, '')
- return f
-
-def stage3Clang(use_compiler, slaves, config_options=''):
- return { 'name' : use_compiler + '-stage3',
- 'builddir' : 'build.'+ use_compiler + '-stage3',
- 'factory' : HostStage3Clang(config_options),
- 'slavenames' : slaves,
- 'category' : 'clang',
- 'properties' : {'use_builder': use_compiler,
- 'compiler_type': 'Release+Asserts',
- 'compiler_built': 'clang-build'}}
-
-def gccTestSuite(use_compiler, slaves, config_options=''):
- return { 'name' : 'gccTestSuite-'+ use_compiler,
- 'builddir' : 'build.'+ 'gccTestSuite-'+ use_compiler,
- 'factory' : gccRunSuite(config_options),
- 'slavenames' : slaves,
- 'category' : 'clang',
- 'properties' : { 'use_builder': use_compiler}}
-
-def boost(tag, use_compiler, slaves, config_options=''):
- return { 'name' : 'boost-' + tag + '-' + use_compiler,
- 'builddir' : 'build.'+ 'boost-' + tag + '-' + use_compiler,
- 'factory' : runboost(config_options),
- 'slavenames' : slaves,
- 'category' : 'clang',
- 'properties' : {'use_builder': use_compiler,
- 'boost_tag': tag}}
-
-def gccRunSuite(config_options):
- f = buildbot.process.factory.BuildFactory()
- # Determine the build directory.
- getBuildDir(f)
- setProperty(f, 'use_path', WithProperties('%(builddir)s/clang-host/bin'))
- cleanCompilerDir(f)
- # Pull test-suite.
- pullclang_tests(f)
- # Download compiler from upstream builder.
- GetCompilerArtifacts(f)
- # Run gcc test suite.
- # TODO: This step returns as a failure because it does not handle expected failures.
-# f.addStep(buildbot.steps.shell.ShellCommand(
-# name='make.check',
-# command=['make', 'check',
-# WithProperties('CC_UNDER_TEST=%(use_path)s/clang'),
-# WithProperties('CXX_UNDER_TEST=%(use_path)s/clang++')],
-# haltOnFailure=True,
-# description=['make check'],
-# env={'PATH': WithProperties('/usr/local/bin/:%(use_path)s:${PATH}')},
-# workdir='test-suite/gcc-4_2-testsuite'))
- return f
-
-def runboost(config_options):
- f = buildbot.process.factory.BuildFactory()
- # Determine the build directory.
- getBuildDir(f)
- f = setProperty(f, 'use_path', WithProperties('%(builddir)s/clang-host/bin'))
- cleanCompilerDir(f)
- # Pull Boost's test-suite.
- pullboostrunner(f)
- # Download compiler artifacts to be used for this build.
- GetCompilerArtifacts(f)
- # Run boost test suite.
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='user-config.jam',
- command=['echo', 'using', 'clang', ':', 'darwin-4.2.1', ':',
- WithProperties('%(use_path)s/clang'), ':', config_options,
- ';', '>', 'user-config.jam'],
- haltOnFailure=True,
- description=['create user-config.jam'],
- workdir=WithProperties('%(builddir)s')))
- f.addStep(buildbot.steps.shell.ShellCommand(
- name='run.py',
- command=['python', 'boost_runner/run.py', WithProperties('--tag=%(boost_tag)s'),
- '--runner=llvmlab', '--bjam-options=--toolset=clang-darwin',
- WithProperties('--bjam-options=--user-config=%(builddir)s/userconfig.jam'),
- '--ftp=ftp://boost:4peiV8Xwxfv9@ftp.siliconman.net',
- WithProperties('--bjam-options=-j%(jobs)s'),'--user=""'],
- haltOnFailure=True,
- description=['boost regression harness'],
- workdir=WithProperties('%(builddir)s'),
- timeout=14400))
- return f
-
-def get_builders(all_slaves):
- phase1 = 'clang-x86_64-osx10-gcc42-RA'
- final_reference = 'clang-x86_64-osx10-RA'
- typeDA = 'Debug+Asserts'
- typeR = 'Release'
- typeRA = 'Release+Asserts'
- phase1_slave = 'llvmlab.local'
- phaseRunners = [phase1_slave]
- phase3_slaves = ['lab-mini-04.local']
- return [
- # This builder should announce good builds and prepare potential release candidates.
- { 'name' : 'Validated Build',
- 'factory' : PublishGoodBuild(),
- 'slavenames' : phaseRunners,
- 'category' : 'status'},
-
- # These builds coordinate and gate each phase as part of the staged design.
- makePhaseBuilder('phase1 - sanity', 'doPhase1','phase2', phaseRunners),
- makePhaseBuilder('phase2 - living', 'doPhase2','phase3', phaseRunners),
- makePhaseBuilder('phase3 - tree health', 'doPhase3','phase4', phaseRunners),
- makePhaseBuilder('phase4 - validation', 'doPhase4','GoodBuild', phaseRunners),
-
- # These are phase 1 build(s).
- stage1Clang(phase1, typeRA, phase1_slave),
-
- # These are phase 2 builds.
- HostedClang ('clang-x86_64-osx10-DA', typeDA, phase1, ['lab-mini-01.local']),
- HostedClang (final_reference, typeRA, phase1, ['lab-mini-02.local'],
- '--enable-optimized'),
- Nightly(phase1, ['lab-mini-03.local']),
-
- # These are phase 3 builds.
- HostedClang ('clang-i386-osx10-RA', typeRA, phase1, phase3_slaves,
- '--enable-optimized', '--target=i386'),
- Nightly('clang-x86_64-osx10-DA', phase3_slaves),
- Nightly(final_reference, phase3_slaves),
- Nightly(final_reference, phase3_slaves, '-O0'),
- Nightly(final_reference, phase3_slaves, '-Os'),
- Nightly(final_reference, phase3_slaves, '-O3'),
- Nightly(final_reference, phase3_slaves, '-flto'),
- Nightly(final_reference, phase3_slaves, '-g'),
-
- # These are phase 4 builds.
- Nightly('clang-i386-osx10-RA', phase3_slaves),
- stage3Clang(final_reference, phase3_slaves),
- gccTestSuite(final_reference, phase3_slaves),
- boost('trunk', final_reference, phase3_slaves)]
-
-def prioritizeBuilders(buildmaster, builders):
- builderPriorities = {
- 'phase1 - sanity':0,
- 'clang-x86_64-osx10-gcc42-RA':0,
- 'phase2 - living':1,
- 'nightly_clang-x86_64-osx10-gcc42-RA':1,
- 'clang-x86_64-osx10-RA':1,
- 'clang-x86_64-osx10-DA':1,
- 'phase3 - tree health':2,
- 'clang-i386-osx10-RA':2,
- 'nightly_clang-x86_64-osx10-DA':3,
- 'nightly_clang-x86_64-osx10-RA':3,
- 'nightly_clang-x86_64-osx10-RA-O0':3,
- 'nightly_clang-x86_64-osx10-RA-Os':3,
- 'nightly_clang-x86_64-osx10-RA-O3':3,
- 'nightly_clang-x86_64-osx10-RA-flto':3,
- 'nightly_clang-x86_64-osx10-RA-g':3,
- 'phase4 - validation':4,
- 'nightly_clang-i386-osx10-RA':5,
- 'clang-x86_64-osx10-RA-stage3':4,
- 'gccTestSuite-clang-x86_64-osx10-RA':4,
- 'nightly_clang-x86_64-osx10-RA-stage3-g':5,
- 'boost-trunk-clang-x86_64-osx10-RA':6,
- 'Validated Build':7}
- builders.sort(key=lambda b: builderPriorities.get(b.name, 0))
- return builders
+import config
+from zorg.buildbot.PhasedBuilderUtils import getPhaseBuilderFactory, PublishGoodBuild
+# Load the phase information.
+import phase_config
+reload(phase_config)
+from phase_config import phases
+
+def get_builders():
+ phaseRunners = ['macpro1']
+ # This builder should announce good builds and prepare potential release
+ # candidates.
+ yield { 'name' : 'Validated Build', 'factory' : PublishGoodBuild(),
+ 'slavenames' : phaseRunners, 'category' : 'status'}
+ # These builds coordinate and gate each phase as part of the staged design.
+ for phase in phases:
+ if phase is phases[-1]:
+ next_phase = 'GoodBuild'
+ else:
+ next_phase = 'phase%d' % (phase['number'] + 1)
+ # Split the phase builders into separate stages.
+ split_stages = config.schedulers.get_phase_stages(phase)
+ yield { 'name' : 'phase%d - %s' % (phase['number'], phase['name']),
+ 'factory' : getPhaseBuilderFactory(config, phase, next_phase,
+ split_stages),
+ 'slavenames' : phaseRunners, 'category' : 'status'}
+ # Add the builders for each phase.
+ import builderconstruction
+ for phase in phases:
+ for info in phase['builders']:
+ builder = builderconstruction.construct(info['name'])
+ builder['category'] = info['category']
+ builder['slavenames'] = list(info['slaves'])
+ if builder.has_key('properties'):
+ props = builder['properties']
+ props ['category'] = info['category']
+ builder['properties'] = props
+ else:
+ builder['properties'] = {'category': info['category']}
+ yield builder
Modified: zorg/trunk/buildbot/llvmlab/master/config/local.cfg
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/local.cfg?rev=170357&r1=170356&r2=170357&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/local.cfg (original)
+++ zorg/trunk/buildbot/llvmlab/master/config/local.cfg Mon Dec 17 13:36:47 2012
@@ -1,4 +1,14 @@
[Master Options]
+# Set when the master is running in a production environment.
+is_production=False
+rsync_user=buildmaster
+master_name=lab.llvm.org
+master_url=http://lab.llvm.org:8013
+master_protocol=http
+
+# Email address for sender of buildbot notifications.
+from_email=llvm.buildmaster at lab.llvm.org
+
# Someone who is automatically CC'ed on all failure messages and on
# failed email lookups.
default_email=david_dean at apple.com
Added: zorg/trunk/buildbot/llvmlab/master/config/phase_config.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/phase_config.py?rev=170357&view=auto
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/phase_config.py (added)
+++ zorg/trunk/buildbot/llvmlab/master/config/phase_config.py Mon Dec 17 13:36:47 2012
@@ -0,0 +1,133 @@
+"""
+Declarative definition of the CI phase organization
+"""
+
+def builder(name, category, slaves):
+ return { 'name' : name,
+ 'category' : category,
+ 'slaves' : slaves }
+
+def build(name, slaves):
+ return builder(name, 'build-public', slaves)
+def test(name, slaves):
+ return builder(name, 'test-public', slaves)
+def experimental(name, slaves):
+ return builder(name, 'experimental', slaves)
+
+# FIXME: Eliminate description from builder name?
+phases = []
+phaseRunners = ['macpro1']
+# if any darwin11 slaves stop working, remember to fix the authorization settings
+# so that gdb will work properly by adding to group procmod with:
+# sudo dscl localhost -append /Local/Default/Groups/procmod GroupMembership [username]
+# also make sure the slave is runninf with an effective group of procmod in the
+# LauchDaemon plist
+
+phase1_slaves=['xserve5']
+
+phases.append(
+ { 'number' : 1,
+ 'name' : 'sanity',
+ 'title' : 'Sanity',
+ 'builders' : [build('clang-x86_64-darwin11-nobootstrap-RA',
+ phase1_slaves)],
+ 'description' : """\
+
+The first phase is responsible for quickly testing that tree is sane -- namely
+that it can be built and that the basic tests are passing. The purpose of this
+phase is to make sure the tree is in good enough condition that most developers
+can get work done, and that it is worth doing further testing.
+
+This phase is also responsible for building the initial Stage 1 compilers which
+will be used to boot strap subsequent builds.
+
+The first phase is targeted to run on almost every commit and to react within at
+most 10 to 15 minutes to failures.""" })
+
+phase2_slaves=['xserve2']
+
+phases.append(
+ { 'number' : 2,
+ 'name' : 'living',
+ 'title' : 'Living On',
+ 'builders' : [build('clang-x86_64-darwin11-DA',
+ phase2_slaves),
+ build('clang-x86_64-darwin11-RA',
+ phase2_slaves),
+# test('lnt_clang-x86_64-darwin11-nobootstrap-RA_x86_64-O3',
+# phase2_slaves),
+ ],
+ 'description' : """\
+The second phase is designed to test that the compiler is basically functional
+and that it is suitable for living on. This means that almost all developers can
+get their work done using sources which have passed this phase.
+
+This phase produces the majority of the compilers which are used in subsequent
+testing.
+
+The second phase is targeted to run on most commits and to react within at most
+15 to 30 minutes to failures.""" })
+
+###
+
+# Phase 3
+
+phase3_slaves = ['xserve3']
+phase3_builders = []
+
+# Add an i386 build.
+phase3_builders.append(build('clang-i386-darwin11-RA', phase3_slaves))
+
+# Add a release (no asserts) build.
+phase3_builders.append(build('clang-x86_64-darwin11-R', phase3_slaves))
+
+# Add a reasonable matrix of nightlies on the final reference compiler.
+# for arch in ('i386', 'x86_64'):
+# for flags in (['-O0','-g'],
+# ['-Os','-g'],
+# ['-O3']):
+# phase3_builders.append(
+# test('lnt_clang-x86_64-darwin11-RA_%s%s' % (arch,
+# ''.join(flags)),
+# phase3_slaves))
+phases.append(
+ { 'number' : 3,
+ 'name' : 'tree health',
+ 'title' : 'Tree Health',
+ 'builders' : phase3_builders,
+ 'description' : """\
+The third phase is designed to check the general health of the tree across a
+variety of targets and build environments. In general, all developers should be
+able to work using sources which have passed this phase, and the tree should be
+good enough for many users.
+
+The third phase is targeted to react within at most 1 to 2 hours.""" })
+
+###
+
+# Phase 4
+
+phase4_slaves = ['xserve4']
+
+phases.append(
+ { 'number' : 4,
+ 'name' : 'validation',
+ 'title' : 'Validation',
+ 'builders' : [
+ # Test the compilers produced in phase 3.
+# test('lnt_clang-i386-darwin11-RA_i386-O3', phase4_slaves),
+# test('lnt_clang-x86_64-darwin11-R_i386-O0', phase4_slaves),
+# test('lnt_clang-x86_64-darwin11-RA_x86_64-Os', phase3_slaves),
+# test('lnt_clang-i386-darwin11-RA_i386-g', phase4_slaves),
+
+ # Test the -DA compiler produced in phase 2. We delay this
+ # to phase 4 because testing a -DA compiler is *very*
+ # slow.
+# test('lnt_clang-x86_64-darwin11-DA_x86_64-O3', phase4_slaves),
+ experimental('lldb_clang-x86_64-darwin11-RA', phase4_slaves),
+ ],
+
+ 'description' : """\
+The fourth and final phase is designed to validate that the tree is in a good
+enough state for a general release."""})
+
Modified: zorg/trunk/buildbot/llvmlab/master/config/schedulers.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/schedulers.py?rev=170357&r1=170356&r2=170357&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/schedulers.py (original)
+++ zorg/trunk/buildbot/llvmlab/master/config/schedulers.py Mon Dec 17 13:36:47 2012
@@ -1,58 +1,89 @@
-from buildbot.scheduler import Scheduler
+from buildbot.schedulers import basic
from buildbot.schedulers import triggerable
from buildbot.process.properties import WithProperties
+from buildbot.changes.filter import ChangeFilter
-def get_schedulers():
+# Load the phase information.
+from phase_config import phases
+
+def get_phase_stages(phase):
+ """get_phase_stages() -> [(normal builders, experimental builders), ...]
+
+ Split a phase's builders into the list of serial stages, and separate
+ experimental builders from non-exerpeimntal ones."""
+
+ builders = dict((b['name'], b)
+ for b in phase['builders'])
+
+ # Each entry in the stage parameter should be a list of builder names.
+ stages = []
+ for stage in phase.get('stages', []):
+ stages.append([builders.pop(name)
+ for name in stage])
+
+ # Add any remaining builders to the final stage.
+ stages.append(builders.values())
+
+ # Split the builder types.
+ split_stages = []
+ for stage in stages:
+ normal_builders = []
+ experimental_builders = []
+ for b in stage:
+ if b['category'] != 'experimental':
+ normal_builders.append(b)
+ else:
+ experimental_builders.append(b)
+ split_stages.append( (normal_builders, experimental_builders) )
- vcScheduler = Scheduler(name='all',branch=None,
- treeStableTimer=2*60,
- builderNames=['phase1 - sanity',])
- startphase1 = triggerable.Triggerable(name='doPhase1',
- builderNames=['clang-x86_64-osx10-gcc42-RA',])
-
- gate1 = triggerable.Triggerable(name='phase2',
- builderNames=['phase2 - living',],
- properties = {'revision':WithProperties('%(got_revision)s')})
- startphase2 = triggerable.Triggerable(name='doPhase2',
- builderNames=[
- 'nightly_clang-x86_64-osx10-gcc42-RA',
- 'clang-x86_64-osx10-DA',
- 'clang-x86_64-osx10-RA',
- ],
- properties = {'revision':WithProperties('%(got_revision)s')})
-
- gate2 = triggerable.Triggerable(name='phase3',
- builderNames=['phase3 - tree health',],
- properties = {'revision':WithProperties('%(got_revision)s')})
- startphase3 = triggerable.Triggerable(name='doPhase3',
- builderNames=[
- 'clang-i386-osx10-RA',
- 'nightly_clang-x86_64-osx10-DA',
- 'nightly_clang-x86_64-osx10-RA',
- 'nightly_clang-x86_64-osx10-RA-O0',
- 'nightly_clang-x86_64-osx10-RA-Os',
- 'nightly_clang-x86_64-osx10-RA-O3',
- 'nightly_clang-x86_64-osx10-RA-g',
- 'nightly_clang-x86_64-osx10-RA-flto',
- ],
- properties = {'revision':WithProperties('%(got_revision)s')})
-
- gate3 = triggerable.Triggerable(name='phase4',
- builderNames=['phase4 - validation',],
- properties = {'revision':WithProperties('%(got_revision)s')})
- startphase4 = triggerable.Triggerable(name='doPhase4',
- builderNames=[
- 'clang-x86_64-osx10-RA-stage3',
- 'nightly_clang-i386-osx10-RA',
- 'gccTestSuite-clang-x86_64-osx10-RA',
- 'boost-trunk-clang-x86_64-osx10-RA',
- ],
- properties = {'revision':WithProperties('%(got_revision)s')})
-
- LastOne = triggerable.Triggerable(name='GoodBuild',
- builderNames=['Validated Build',],
- properties = {'revision':WithProperties('%(got_revision)s')})
-
- return [vcScheduler, startphase1, gate1, startphase2, gate2,
- startphase3, gate3, startphase4, LastOne, ]
+ return split_stages
+
+def get_schedulers():
+ first_phase = phases[0]
+ last_phase = phases[-1]
+ # The VC scheduler initiates the first phase.
+ # Each phase, in turn, triggers the next phase,
+ # until the fianl phase
+
+ for phase in phases:
+ phase_name = 'phase%d' % phase['number']
+ my_filter = ChangeFilter(category = phase_name)
+ if phase == first_phase:
+ delay=120
+ else:
+ delay=15
+
+ yield basic.AnyBranchScheduler(
+ name = phase_name, treeStableTimer=delay,
+ change_filter = my_filter,
+ builderNames = ['phase%d - %s' % (phase['number'], phase['name'])],
+ )
+
+ # Add triggers for initiating the builds in each phase.
+ for phase in phases:
+
+ # Split the phase builders into separate stages.
+ split_stages = get_phase_stages(phase)
+ for i, (normal, experimental) in enumerate(split_stages):
+ # Add the normal trigger, if used.
+ if normal:
+ yield triggerable.Triggerable(
+ name = 'phase%d-stage%d' % (phase['number'], i),
+ builderNames = [b['name'] for b in normal])
+
+ # Add the experimental trigger, if used.
+ if experimental:
+ yield triggerable.Triggerable(
+ name = 'phase%d-stage%d-experimental' % (phase['number'],
+ i),
+ builderNames = [b['name'] for b in experimental])
+
+ # Add a final trigger to trigger the validated build scheduler.
+ phase_name = 'GoodBuild'
+ my_filter = ChangeFilter(category = phase_name)
+ yield basic.AnyBranchScheduler(
+ name = phase_name, treeStableTimer=5,
+ builderNames = ['Validated Build',],
+ change_filter = my_filter,
+ )
Modified: zorg/trunk/buildbot/llvmlab/master/config/slaves.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/slaves.py?rev=170357&r1=170356&r2=170357&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/slaves.py (original)
+++ zorg/trunk/buildbot/llvmlab/master/config/slaves.py Mon Dec 17 13:36:47 2012
@@ -1,30 +1,33 @@
import buildbot
import buildbot.buildslave
import os
+import config
-def create_slave(name, *args, **kwargs):
- return buildbot.buildslave.BuildSlave(name, password='password',
- *args, **kwargs)
+from zorg.buildbot.PhasedBuilderUtils import set_config_option
+
+def create_slave(name, jobs, max_builds = None):
+ if max_builds is None:
+ max_builds = jobs // 2
+ return buildbot.buildslave.BuildSlave(
+ name, password = 'password',
+ notify_on_missing = set_config_option('Master Options',
+ 'default_email',
+ 'david_dean at apple.com'),
+ properties = { 'jobs' : jobs },
+ max_builds = 1)
def get_build_slaves():
- #phase runners
- yield create_slave("llvmlab.local",
- notify_on_missing="david_dean at apple.com",
- properties = { 'jobs' : 16 },
- max_builds = 16)
- yield create_slave("lab-mini-01.local",
- notify_on_missing="david_dean at apple.com",
- properties = { 'jobs' : 2 },
- max_builds = 1)
- yield create_slave("lab-mini-02.local",
- notify_on_missing="david_dean at apple.com",
- properties = { 'jobs' : 2 },
- max_builds = 1)
- yield create_slave("lab-mini-03.local",
- notify_on_missing="david_dean at apple.com",
- properties = { 'jobs' : 2 },
- max_builds = 1)
- yield create_slave("lab-mini-04.local",
- notify_on_missing="david_dean at apple.com",
- properties = { 'jobs' : 2 },
- max_builds = 1)
+ # Phase runnner.
+ yield create_slave('macpro1', jobs = 1, max_builds = 8)
+
+ # Builders.
+ yield create_slave('xserve2', jobs = 4, max_builds = 2)
+ yield create_slave('xserve3', jobs = 4, max_builds = 2)
+ yield create_slave('xserve4', jobs = 4, max_builds = 2)
+ yield create_slave('xserve5', jobs = 4, max_builds = 2)
+
+ has_production = config.options.has_option('Master Options', 'is_production')
+ is_production = config.options.has_option('Master Options', 'is_production')
+ if has_production and is_production:
+ # Test slave which can do anything.
+ yield create_slave('localhost', 8)
Modified: zorg/trunk/buildbot/llvmlab/master/config/status.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/status.py?rev=170357&r1=170356&r2=170357&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/status.py (original)
+++ zorg/trunk/buildbot/llvmlab/master/config/status.py Mon Dec 17 13:36:47 2012
@@ -1,30 +1,102 @@
-import os
import buildbot
import buildbot.status.html
import buildbot.status.mail
import buildbot.status.words
-
import config
+import os
+
+from zorg.buildbot.PhasedBuilderUtils import set_config_option
from zorg.buildbot.util.ConfigEmailLookup import ConfigEmailLookup
+from zorg.buildbot.util.InformativeMailNotifier import InformativeMailNotifier
+
def get_status_targets(standard_builders):
- default_email = config.options.get('Master Options', 'default_email')
+ # Get from/to email addresses.
+ from_email = set_config_option('Master Options', 'from_email')
+ default_email = set_config_option('Master Options', 'default_email')
+
+ # Check whether we are in testing mode, if so, just add minimal and verbose
+ # status clients.
+ if True:
+ return [
+ buildbot.status.html.WebStatus(
+ http_port = 8013, allowForce = True),
+
+ InformativeMailNotifier(fromaddr = from_email,
+ extraRecipients = ['david_dean at apple.com'],
+ sendToInterestedUsers = False,
+ mode = 'change',
+ addLogs = False,
+ num_lines = 15)]
+
+ # Get the path to the authors file we use for email lookup.
+ llvm_authors_path = os.path.join(os.path.dirname(__file__),
+ set_config_option('Master Options',
+ 'llvm_authors_path'))
+
+ # Construct a lookup object to be used for public builders.
+ public_lookup = ConfigEmailLookup(
+ llvm_authors_path, default_address = 'llvm-testresults at cs.uiuc.edu')
+
return [
buildbot.status.html.WebStatus(
http_port = 8013, allowForce = True),
- buildbot.status.mail.MailNotifier(
- fromaddr = 'david_dean at apple.com',
- extraRecipients = ['daniel_dunbar at apple.com','david_dean at apple.com'],
- sendToInterestedUsers=False,
+ buildbot.status.words.IRC('irc.oftc.net', 'llvmlab',
+ port=6668,
+ channels=['llvm'],
+ allowForce=False,
+ password='smooshy',
+ notify_events=['successToFailure', 'failureToSuccess'],
+ categories=['build', 'test']),
+
+ # Experimental failing build notifier.
+ #
+ # These emails only go to the catch-all list.
+ InformativeMailNotifier(
+ fromaddr = from_email,
+ extraRecipients = ['llvm-testresults at cs.uiuc.edu'],
+ sendToInterestedUsers = False,
+ mode = 'failing',
+ categories = ['experimental'],
+ addLogs = False,
+ num_lines = 15),
+
+ # Regular problem build notifier.
+ #
+ # These emails go to the interested (internal users), and the catch-all
+ # list.
+ InformativeMailNotifier(
+ fromaddr = from_email,
+ lookup = internal_lookup,
+ extraRecipients = ['llvm-testresults at cs.uiuc.edu'],
+ sendToInterestedUsers = True,
mode = 'problem',
- relayhost="mail-in2.apple.com",
- categories=['clang','tests'],
- ),
+ categories = ['build', 'test'],
+ addLogs = False,
+ num_lines = 15),
+
+ # Regular failing build notifier.
+ #
+ # These emails only go to the catch-all list.
+ #
+ # FIXME: Eventually, these should also go to the current build czars.
+ # TODO: change subject to differentiate these from the problem emails
+ InformativeMailNotifier(
+ fromaddr = from_email,
+ sendToInterestedUsers = False,
+ extraRecipients = ['llvm-testresults at cs.uiuc.edu'],
+ mode = 'failing',
+ categories = ['build', 'test'],
+ addLogs = False,
+ num_lines = 15),
+
+ # Phase status change notifier.
+ #
+ # These emails only go to the catch-all list.
buildbot.status.mail.MailNotifier(
- fromaddr = 'david_dean at apple.com',
- extraRecipients = ['david_dean at apple.com'],
- sendToInterestedUsers=False,
+ fromaddr = from_email,
+ sendToInterestedUsers = False,
+ extraRecipients = ['llvm-testresults at cs.uiuc.edu'],
mode = 'change',
- relayhost="mail-in2.apple.com",
- categories=['status'],
- ),]
+ categories = ['status']),]
+
Modified: zorg/trunk/buildbot/llvmlab/master/master.cfg
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/master.cfg?rev=170357&r1=170356&r2=170357&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/master.cfg (original)
+++ zorg/trunk/buildbot/llvmlab/master/master.cfg Mon Dec 17 13:36:47 2012
@@ -5,14 +5,30 @@
Top level build bot configuration file.
"""
-# Expect llvm_zorg to be checked out at the top level.
+# Extend paths to allow loading zorg and config modules.
+# handle either llvm_zorg or zorg to be checked out at the top level.
import os, sys
-path = os.path.join(os.environ.get('HOME'), 'llvm_zorg', 'buildbot')
+path = os.path.join(os.environ.get('HOME'), 'zorg', 'buildbot',
+ 'llvmlab', 'master')
if path not in sys.path:
- sys.path.append(path)
-path = os.path.join(os.environ.get('HOME'), 'llvm_zorg')
-if path not in sys.path:
- sys.path.append(path)
+ sys.path.append(path)
+for zorg in ['zorg', 'llvm_zorg']:
+ path = os.path.join(os.environ.get('HOME'), zorg, 'buildbot')
+ if path not in sys.path:
+ sys.path.append(path)
+ path = os.path.join(os.environ.get('HOME'), zorg)
+ if path not in sys.path:
+ sys.path.append(path)
+
+# Reload every module, so that we are more resilient to code changes
+# on buildbot reconfig.
+#
+# There are still possible problems related to the ordering dependency
+# on reloads, but this should at least help.
+from zorg.buildbot.util import reloading
+reloading.reload_all(only_paths = [os.environ.get('HOME'), basedir,
+ os.path.normpath(basedir)],
+ log = True)
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
@@ -20,6 +36,8 @@
import config
import config.schedulers
+import config.slaves
+import config.status
# Reload stuff automatically.
#
@@ -48,31 +66,14 @@
# about source code changes. Any class which implements IChangeSource can be
# put here: there are several in buildbot/changes/*.py to choose from.
-import os
-kStableTimeout = 2 * 60
-
-import buildbot.changes.svnpoller
from buildbot.changes.pb import PBChangeSource
-class LLVMPoller(buildbot.changes.svnpoller.SVNPoller):
- def __init__(self, project, pollinterval=120, histmax=10):
- buildbot.changes.svnpoller.SVNPoller.__init__(self,
- project=project,
- svnurl='http://llvm.org/svn/llvm-project/%s/trunk' % project,
- pollinterval=pollinterval,
- histmax=histmax,
- revlinktmpl='http://llvm.org/viewvc/llvm-project/?view=rev&revision=%s')
-c['change_source'] = []
-if True:
- c['change_source'].append(LLVMPoller("llvm"))
- c['change_source'].append(LLVMPoller("cfe"))
-# c['change_source'].append(LLVMPoller("llvm-gcc-4.2/trunk"))
- c['change_source'].append(LLVMPoller("test-suite"))
+c['change_source'] = [PBChangeSource()]
####### BUILDERS
slaves_name = [s.slavename for s in c['slaves']]
-c['builders'] = builders = list(config.builders.get_builders(slaves_name) )
+c['builders'] = builders = list(config.builders.get_builders() )
####### STATUS TARGETS
More information about the llvm-commits
mailing list