[zorg] r254717 - [zorg] Add support for uploading artifacts to the 'llvmlab bisect' bucket and enable this for clang-cmake-mips.
Daniel Sanders via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 4 06:30:39 PST 2015
Author: dsanders
Date: Fri Dec 4 08:30:39 2015
New Revision: 254717
URL: http://llvm.org/viewvc/llvm-project?rev=254717&view=rev
Log:
[zorg] Add support for uploading artifacts to the 'llvmlab bisect' bucket and enable this for clang-cmake-mips.
Summary:
Adds a getClangCMakeGCSBuildFactory() based on getClangCMakeBuildFactory(). This
function adds a new keyword argument to getClangCMakeBuildFactory():
stage1_upload_directory - This is the bucket subdirectory to upload to and
should match the builder name.
The buildslave must have gsutil installed to use this argument and must have an
appropriate config installed. If the config is not installed in gsutil's search
path then the BOTO_CONFIG envvar can be used to specify it. Additionally the
BUCKET envvar can be used (as per 'llvmlab bisect') to specify an alternate GCS
bucket to use.
Reviewers: gkistanova
Subscribers: chatur01, llvm-commits, cmatthews, gkistanova
Differential Revision: http://reviews.llvm.org/D14866
Modified:
zorg/trunk/buildbot/osuosl/master/config/builders.py
zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
Modified: zorg/trunk/buildbot/osuosl/master/config/builders.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/builders.py?rev=254717&r1=254716&r2=254717&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/builders.py (original)
+++ zorg/trunk/buildbot/osuosl/master/config/builders.py Fri Dec 4 08:30:39 2015
@@ -713,7 +713,9 @@ def _get_sanitizer_builders():
checkout_compiler_rt=True,
extra_cmake_args=["-DLLVM_HOST_TRIPLE=mips-unknown-linux-gnu",
"-DLLVM_DEFAULT_TARGET_TRIPLE=mips-unknown-linux-gnu",
- "-DLLVM_TARGET_ARCH=Mips"])},
+ "-DLLVM_TARGET_ARCH=Mips"],
+ stage1_upload_directory='clang-cmake-mips',
+ env = {'BOTO_CONFIG': '/var/buildbot/llvmlab-build-artifacts.boto'},
]
def _get_openmp_builders():
Modified: zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/ClangBuilder.py?rev=254717&r1=254716&r2=254717&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/ClangBuilder.py (original)
+++ zorg/trunk/zorg/buildbot/builders/ClangBuilder.py Fri Dec 4 08:30:39 2015
@@ -2,6 +2,7 @@ import buildbot
import buildbot.process.factory
import copy
import os
+from datetime import datetime
from buildbot.process.properties import WithProperties, Property
from buildbot.steps.shell import Configure, ShellCommand, SetProperty
@@ -481,6 +482,99 @@ def addSVNUpdateSteps(f,
defaultBranch='trunk',
workdir='test/test-suite'))
+def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env,
+ gcs_url_property=None):
+ """
+ Add steps to upload to the Google Cloud Storage bucket.
+
+ f - The BuildFactory to modify.
+ package_name - The name of this package for the descriptions (e.g.
+ 'stage 1')
+ install_prefix - The directory the build has been installed to.
+ gcs_directory - The subdirectory of the bucket root to upload to. This
+ should match the builder name.
+ env - The environment to use. Set BOTO_CONFIG to use a configuration file
+ in a non-standard location, and BUCKET to use a different GCS bucket.
+ gcs_url_property - Property to assign the GCS url to.
+ """
+
+ gcs_url_fmt = ('gs://%(gcs_bucket)s/%(gcs_directory)s/'
+ 'clang-r%(got_revision)s-t%(now)s-b%(buildnumber)s.tar.xz')
+ time_fmt = '%Y-%m-%d_%H-%M-%S'
+ gcs_url = \
+ WithProperties(
+ gcs_url_fmt,
+ gcs_bucket=lambda _: env.get('BUCKET', 'llvm-build-artifacts'),
+ gcs_directory=lambda _: gcs_directory,
+ now=lambda _: datetime.utcnow().strftime(time_fmt))
+
+ if gcs_url_property:
+ f.addStep(SetProperty(
+ name="record GCS url for " + package_name,
+ command=['echo', gcs_url],
+ property=gcs_url_property))
+
+ f.addStep(ShellCommand(name='package ' + package_name,
+ command=['tar', 'cvfJ', '../install.tar.xz', '.'],
+ description='packaging ' + package_name + '...',
+ workdir=install_prefix,
+ env=env))
+
+ f.addStep(ShellCommand(
+ name='upload ' + package_name + ' to storage bucket',
+ command=['gsutil', 'cp', '../install.tar.xz', gcs_url],
+ description=('uploading ' + package_name +
+ 'to storage bucket ...'),
+ workdir=install_prefix,
+ env=env))
+
+def getClangCMakeGCSBuildFactory(
+ clean=True,
+ test=True,
+ cmake='cmake',
+ jobs=None,
+
+ # VS tools environment variable if using MSVC. For example,
+ # %VS120COMNTOOLS% selects the 2013 toolchain.
+ vs=None,
+ vs_target_arch='x86',
+
+ # Multi-stage compilation
+ useTwoStage=False,
+ testStage1=True,
+ stage1_config='Release',
+ stage2_config='Release',
+
+ # Test-suite
+ runTestSuite=False,
+ nt_flags=[],
+ submitURL=None,
+ testerName=None,
+
+ # Environmental variables for all steps.
+ env={},
+ extra_cmake_args=[],
+
+ # Extra repositories
+ checkout_clang_tools_extra=True,
+ checkout_compiler_rt=True,
+
+ # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
+ stage1_upload_directory=None,
+
+ # Triggers
+ trigger_after_stage1=[]):
+ return _getClangCMakeBuildFactory(
+ clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
+ vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
+ testStage1=testStage1, stage1_config=stage1_config,
+ stage2_config=stage2_config, runTestSuite=runTestSuite,
+ nt_flags=nt_flags, submitURL=submitURL, testerName=testerName,
+ env=env, extra_cmake_args=extra_cmake_args,
+ checkout_clang_tools_extra=checkout_clang_tools_extra,
+ checkout_compiler_rt=checkout_compiler_rt,
+ stage1_upload_directory=stage1_upload_directory,
+ trigger_after_stage1=trigger_after_stage1)
def getClangCMakeBuildFactory(
clean=True,
@@ -512,6 +606,52 @@ def getClangCMakeBuildFactory(
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True):
+ return _getClangCMakeBuildFactory(
+ clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
+ vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
+ testStage1=testStage1, stage1_config=stage1_config,
+ stage2_config=stage2_config, runTestSuite=runTestSuite,
+ nt_flags=nt_flags, submitURL=submitURL, testerName=testerName,
+ env=env, extra_cmake_args=extra_cmake_args,
+ checkout_clang_tools_extra=checkout_clang_tools_extra,
+ checkout_compiler_rt=checkout_compiler_rt)
+
+def _getClangCMakeBuildFactory(
+ clean=True,
+ test=True,
+ cmake='cmake',
+ jobs=None,
+
+ # VS tools environment variable if using MSVC. For example,
+ # %VS120COMNTOOLS% selects the 2013 toolchain.
+ vs=None,
+ vs_target_arch='x86',
+
+ # Multi-stage compilation
+ useTwoStage=False,
+ testStage1=True,
+ stage1_config='Release',
+ stage2_config='Release',
+
+ # Test-suite
+ runTestSuite=False,
+ nt_flags=[],
+ submitURL=None,
+ testerName=None,
+
+ # Environmental variables for all steps.
+ env={},
+ extra_cmake_args=[],
+
+ # Extra repositories
+ checkout_clang_tools_extra=True,
+ checkout_compiler_rt=True,
+
+ # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
+ stage1_upload_directory=None,
+
+ # Triggers
+ trigger_after_stage1=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
@@ -601,13 +741,17 @@ def getClangCMakeBuildFactory(
workdir=stage1_build,
env=env))
- if useTwoStage or runTestSuite:
+ if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
+ if stage1_upload_directory:
+ addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory,
+ env, gcs_url_property='stage1_package_gcs_url')
+
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
More information about the llvm-commits
mailing list