[zorg] r243172 - [zorg] Add lldb build factory that launches scripts on slave machine
Ying Chen
chying at google.com
Fri Jul 24 15:03:41 PDT 2015
Author: chying
Date: Fri Jul 24 17:03:40 2015
New Revision: 243172
URL: http://llvm.org/viewvc/llvm-project?rev=243172&view=rev
Log:
[zorg] Add lldb build factory that launches scripts on slave machine
Summary:
- Add new function getLLDBScriptCommandsFactory, it will launch script file for each step
- Add new builders lldb Windows, and lldb buildserver
- Create triggerable scheduler to trigger android builds
- Add new builders to rotation email monitor list
Reviewers: chaoren, vharron
Subscribers: tberghammer, danalbert, srhines, llvm-commits
Differential Revision: http://reviews.llvm.org/D11445
Modified:
zorg/trunk/buildbot/osuosl/master/config/builders.py
zorg/trunk/buildbot/osuosl/master/config/slaves.py
zorg/trunk/buildbot/osuosl/master/config/status.py
zorg/trunk/buildbot/osuosl/master/master.cfg
zorg/trunk/zorg/buildbot/builders/LLDBBuilder.py
Modified: zorg/trunk/buildbot/osuosl/master/config/builders.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/builders.py?rev=243172&r1=243171&r2=243172&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/builders.py (original)
+++ zorg/trunk/buildbot/osuosl/master/config/builders.py Fri Jul 24 17:03:40 2015
@@ -1040,34 +1040,43 @@ def _get_experimental_scheduled_builders
'slavenames': ["lldb-build1-ubuntu-1404"],
'builddir': "buildWorkingDir",
'category' : 'lldb',
- 'factory': LLDBBuilder.getLLDBUbuntuCMakeBuildFactory(
- build_compiler="clang",
- build_type="Release",
- local_test_archs=['x86_64', 'i386'],
- local_test_compilers=['clang', 'totclang', 'gcc4.8.2', 'gcc4.9.2'],
- #remote_configs=[RemoteConfig('linux','x86_64',['clang'],['x86_64'])],
- env={'SHELL':"/bin/bash"})},
+ 'factory': LLDBBuilder.getLLDBScriptCommandsFactory(
+ downloadBinary=False,
+ buildAndroid=False,
+ runTest=True)},
{'name': "lldb-x86_64-darwin-13.4",
'slavenames': ["lldb-x86_64-darwin-13.4"],
'builddir': "buildDir",
'category' : 'lldb',
- 'factory': LLDBBuilder.getLLDBxcodebuildFactory(
- build_type='Release',
- remote_configs=[RemoteConfig('android', 'arm', ['arm-linux-androideabi-gcc'], ['arm'])],
- env={'SHELL':"/bin/bash"})},
+ 'factory': LLDBBuilder.getLLDBScriptCommandsFactory(
+ downloadBinary=True,
+ buildAndroid=False,
+ runTest=True)},
{'name': "lldb-x86_64-ubuntu-14.04-android",
'slavenames': ["lldb-lab-linux01"],
'builddir': "lldb-cross-compile",
'category' : 'lldb',
- 'factory': LLDBBuilder.getLLDBUbuntuCMakeBuildFactory(
- build_compiler="clang",
- build_type="Release",
- remote_configs= [
- RemoteConfig('android', 'i386', ['i686-linux-android-gcc'], ['i386']),
- RemoteConfig('android', 'arm', ['arm-linux-androideabi-gcc'], ['arm']),
- RemoteConfig('android', 'aarch64', ['aarch64-linux-android-gcc'], ['aarch64'])
- ],
- env={'SHELL':"/bin/bash"})},
+ 'factory': LLDBBuilder.getLLDBScriptCommandsFactory(
+ downloadBinary=True,
+ buildAndroid=False,
+ runTest=True)},
+ {'name': "lldb-windows7-android",
+ 'slavenames': ["lldb-lab-win01"],
+ 'builddir': "lldb-win7-android",
+ 'category': "lldb",
+ 'factory': LLDBBuilder.getLLDBScriptCommandsFactory(
+ downloadBinary=True,
+ buildAndroid=False,
+ runTest=True,
+ scriptExt='.bat')},
+ {'name': "lldb-x86_64-ubuntu-14.04-buildserver",
+ 'slavenames': ["lldb-linux-android-buildserver"],
+ 'builddir': "lldb-android-buildserver",
+ 'category' : 'lldb',
+ 'factory': LLDBBuilder.getLLDBScriptCommandsFactory(
+ downloadBinary=False,
+ buildAndroid=True,
+ runTest=False)}
]
# Builders responsible building Sphinix documentation
Modified: zorg/trunk/buildbot/osuosl/master/config/slaves.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/slaves.py?rev=243172&r1=243171&r2=243172&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/slaves.py (original)
+++ zorg/trunk/buildbot/osuosl/master/config/slaves.py Fri Jul 24 17:03:40 2015
@@ -186,6 +186,10 @@ def get_build_slaves():
# Ubuntu 14.04 x86_64, Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz
create_slave("lldb-lab-linux01", properties={'jobs': 40}, max_builds=1),
+ # Windows 7 x86_64
+ create_slave("lldb-lab-win01", properties={'jobs': 40}, max_builds=1),
+ # Ubuntu 14.04 x86_64
+ create_slave("lldb-linux-android-buildserver", properties={'jobs': 40}, max_builds=1),
# Ubuntu 14.04 x86_64, Intel(R) Xeon(R) CPU L5520 @ 2.27GHz
create_slave("bpf-build-slave01", properties={'jobs': 16}, max_builds=1),
Modified: zorg/trunk/buildbot/osuosl/master/config/status.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/status.py?rev=243172&r1=243171&r2=243172&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/status.py (original)
+++ zorg/trunk/buildbot/osuosl/master/config/status.py Fri Jul 24 17:03:40 2015
@@ -130,6 +130,8 @@ def get_status_targets(standard_builders
subject="Build %(builder)s Failure",
mode = "failing",
builders = ["lldb-x86_64-ubuntu-14.04-cmake",
+ "lldb-x86_64-ubuntu-14.04-buildserver",
+ "lldb-windows7-android",
"lldb-x86_64-darwin-13.4",
"lldb-x86_64-ubuntu-14.04-android"],
addLogs=False,
Modified: zorg/trunk/buildbot/osuosl/master/master.cfg
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/master.cfg?rev=243172&r1=243171&r2=243172&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/master.cfg (original)
+++ zorg/trunk/buildbot/osuosl/master/master.cfg Fri Jul 24 17:03:40 2015
@@ -104,6 +104,7 @@ c['status'] = config.status.get_status_t
####### SCHEDULERS
from buildbot.scheduler import Scheduler
+from buildbot.schedulers import triggerable
from buildbot.schedulers.filter import ChangeFilter
from buildbot.schedulers.basic import SingleBranchScheduler
@@ -169,9 +170,16 @@ c['schedulers'].append(SingleBranchSched
"llvm",
"lld"])))
+# lldb android builders will be triggered by completion of buildserver instead of by source change
+lldb_android_builders=["lldb-x86_64-ubuntu-14.04-android",
+ "lldb-windows7-android",
+ "lldb-x86_64-darwin-13.4"]
+c['schedulers'].append((triggerable.Triggerable(name="lldb_android_scheduler",
+ builderNames=lldb_android_builders)))
+
c['schedulers'].append(SingleBranchScheduler(name="lldb_scheduler",
treeStableTimer=2*60,
- builderNames=get_all_for("lldb"),
+ builderNames=list(set(get_all_for("lldb"))-set(lldb_android_builders)),
change_filter=depends_on([
"llvm",
"cfe",
Modified: zorg/trunk/zorg/buildbot/builders/LLDBBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/LLDBBuilder.py?rev=243172&r1=243171&r2=243172&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/LLDBBuilder.py (original)
+++ zorg/trunk/zorg/buildbot/builders/LLDBBuilder.py Fri Jul 24 17:03:40 2015
@@ -8,6 +8,7 @@ from buildbot.steps.shell import Configu
from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
from buildbot.steps.slave import RemoveDirectory
from buildbot.process.properties import WithProperties, Property
+from buildbot.steps import trigger
import zorg.buildbot.commands.BatchFileDownload as batch_file_download
from zorg.buildbot.commands.LitTestCommand import LitTestCommand
from zorg.buildbot.builders.Util import getVisualStudioEnvironment
@@ -855,3 +856,109 @@ def getLLDBxcodebuildFactory(use_cc=None
#
# Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error. return f
return f
+
+def getShellCommandStep(f,
+ name,
+ command,
+ description="",
+ flunkOnFailure=True,
+ haltOnFailure=True,
+ workdir='scripts',
+ env=None):
+ if env is None:
+ env = {}
+ f.addStep(ShellCommand(name=name,
+ command=command,
+ description=description,
+ env=env,
+ flunkOnFailure=flunkOnFailure,
+ haltOnFailure=haltOnFailure,
+ workdir=workdir))
+
+# get configuration of tests
+# config file should be placed under builddir on builder machine
+# file name: test_cfg.json
+# content: json format with keys test[num]:target-compiler-architecture
+# example: {"test1":"android-gcc4.9-i386",
+# "test2":"local-clang-x86",
+# "test3":"local-clang-i386"}
+def getTestConfig(f):
+ def getRemoteCfg(rc, stdout, stderr):
+ return json.loads(stdout)
+ f.addStep(SetProperty(name="get test config",
+ command="cat test_cfg.json",
+ extract_fn=getRemoteCfg,
+ description="get remote target",
+ workdir="scripts"))
+ return f
+def getTestSteps(f, scriptExt):
+ # buildbot doesn't support dynamic step creation, so create 8 test steps as place holder
+ # then each builder will define available tests in test_cfg.json
+ # if there're less than 8 tests defined on certain builder, extra steps will be skipped and hidden from test details view
+ # **hide step is not supported by buildbot 0.8.5
+ # flunkOnFailure only takes boolean value, and cannot take configurable property.
+ # workaround: don't flunk the last two tests
+ # put non flunkable tests as the last two, test7 and test8
+ getTestConfig(f)
+ for x in range(1, 9):
+ test='test'+str(x)
+ f.addStep(LitTestCommand(name=test,
+ command=['test' + scriptExt,
+ Property(test)],
+ description="",
+ doStepIf=lambda step: step.build.hasProperty(step.name),
+ flunkOnFailure=(x<7),
+ warnOnFailure=(x<7),
+ workdir='scripts'))
+
+def getLLDBScriptCommandsFactory(
+ downloadBinary=True,
+ buildAndroid=False,
+ runTest=True,
+ scriptExt='.sh',
+ ):
+ f = buildbot.process.factory.BuildFactory()
+
+ # Checkout source code
+ getShellCommandStep(f, name='checkout source code',
+ command=['checkoutSource' + scriptExt,
+ WithProperties('%(revision)s')])
+
+ # Set source revision
+ f.addStep(SetProperty(name="set revision",
+ command=['svnversion'],
+ property="got_revision",
+ workdir="llvm"))
+
+ # Configure
+ getShellCommandStep(f, name='cmake local',
+ command=['cmake' + scriptExt])
+
+ # Build
+ getShellCommandStep(f, name='ninja build local',
+ command=['buildLocal' + scriptExt])
+ if buildAndroid:
+ getShellCommandStep(f, name='build android',
+ command=['buildAndroid' + scriptExt])
+
+ # Get lldb-server binaries
+ if downloadBinary:
+ getShellCommandStep(f, name='get lldb-server binaries',
+ command=['downloadBinaries' + scriptExt,
+ WithProperties('%(revision)s')])
+
+ # Test
+ if runTest:
+ getTestSteps(f, scriptExt)
+ # upload test traces
+ getShellCommandStep(f, name='upload test traces',
+ command=['uploadTestTrace' + scriptExt],
+ flunkOnFailure=False)
+
+ # Upload lldb-server binaries and trigger android builders
+ if buildAndroid:
+ getShellCommandStep(f, name='upload lldb-server binaries',
+ command=['uploadBinaries' + scriptExt])
+ f.addStep(trigger.Trigger(schedulerNames=['lldb_android_scheduler'],
+ waitForFinish=False))
+ return f
More information about the llvm-commits
mailing list