[zorg] r189650 - [PEP8] Renamed zorg.buildbot.Artifacts => zorg.buildbot.util.artifacts and performed all necessary renamings.

Michael Gottesman mgottesman at apple.com
Thu Aug 29 22:46:16 PDT 2013


Author: mgottesman
Date: Fri Aug 30 00:46:15 2013
New Revision: 189650

URL: http://llvm.org/viewvc/llvm-project?rev=189650&view=rev
Log:
[PEP8] Renamed zorg.buildbot.Artifacts => zorg.buildbot.util.artifacts and performed all necessary renamings.

Added:
    zorg/trunk/zorg/buildbot/util/artifacts.py
      - copied, changed from r189328, zorg/trunk/zorg/buildbot/Artifacts.py
Removed:
    zorg/trunk/zorg/buildbot/Artifacts.py
Modified:
    zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py
    zorg/trunk/zorg/buildbot/PhasedBuilderUtils.py
    zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
    zorg/trunk/zorg/buildbot/builders/LNTBuilder.py
    zorg/trunk/zorg/buildbot/builders/LibCXXBuilder.py
    zorg/trunk/zorg/buildbot/builders/gccSuiteBuilder.py

Modified: zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py?rev=189650&r1=189649&r2=189650&view=diff
==============================================================================
--- zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py (original)
+++ zorg/trunk/buildbot/llvmlab/master/config/builderconstruction.py Fri Aug 30 00:46:15 2013
@@ -1,5 +1,5 @@
 #from zorg.buildbot.builders.LNTBuilder import CreateLNTNightlyFactory
-from zorg.buildbot.Artifacts import rsync_user, master_name
+from zorg.buildbot.util.artifacts import rsync_user, master_name
 import zorg.buildbot.builders.ClangBuilder as ClangBuilder
 from zorg.buildbot.builders.LLDBBuilder import getLLDBxcodebuildFactory
 import zorg.buildbot.builders.LibCXXBuilder as LibCXXBuilder

Removed: zorg/trunk/zorg/buildbot/Artifacts.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/Artifacts.py?rev=189649&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/Artifacts.py (original)
+++ zorg/trunk/zorg/buildbot/Artifacts.py (removed)
@@ -1,238 +0,0 @@
-import os
-
-import buildbot
-import config
-
-from buildbot.steps.shell import WithProperties
-from zorg.buildbot.PhasedBuilderUtils import setProperty, determine_phase_id
-from zorg.buildbot.PhasedBuilderUtils import set_config_option
-
-# Get some parameters about where to upload and download results from.
-is_production = set_config_option('Master Options', 'is_production')
-if is_production:
-    rsync_user = set_config_option('Master Options', 'rsync_user',
-                                   'buildmaster')
-    master_name = set_config_option('Master Options', 'master_name',
-                                    'localhost')
-    master_protocol = set_config_option('Master Options', 
-                                        'master_protocol', 'http')
-    base_download_url = '%s://%s/artifacts' % (master_protocol, master_name)
-    base_package_url = '%s://%s/packages' % (master_protocol, master_name)
-    package_url = set_config_option('Master Options', 'package_url',
-                                    base_package_url)
-    artifacts_path = set_config_option('Master Options', 'artifacts_path',
-                                       os.path.expanduser('~/artifacts'))
-    curl_flags = set_config_option('Master Options', 'curl_flags',
-                                   '-fvLo')    
-else:
-    import getpass
-    rsync_user = getpass.getuser()
-    master_name = 'localhost'
-    master_protocol = 'http'
-    base_download_url = 'http://%s/~%s/artifacts' % (master_name, rsync_user)
-    package_url = 'http://%s/~%s/packages' % (master_name, rsync_user)
-    artifacts_path = os.path.expanduser('~/artifacts')
-    curl_flags = '-fvLo'
-
-base_rsync_path = '%s@%s:%s' % (rsync_user, master_name, artifacts_path)
-
-# This method is used in determining the name of a given compiler archive
-def _determine_compiler_kind(props):
-    # we need to differentiate between configure/make style builds (clang)
-    # from buildit style builde (apple-clang)
-
-    buildName = props['buildername']
-    kind = buildName
-    subname = buildName
-    if '_' in buildName:
-        kind,subname = buildName.split('_', 1)
-    if 'clang' in kind:
-        subname = kind
-    for kind in ('apple-clang','clang'):
-        if kind in subname:
-            return kind
-    raise ValueError, "unknown compiler"
-
-# compiler_path and archive_name should be completely deterministic. Any 
-# methods acting on an archive should use the following two methods to
-# calculate the path and/or name for an archive
-def _determine_archive_name(props):
-    # phase_id must be set upstream. Usually by a phase builder
-    archive_name = _determine_compiler_kind(props)
-    if props.has_key('phase_id') and props['phase_id']:
-        archive_name += '-' + props['phase_id'] + '.tar.gz'
-    else:
-        raise ValueError, "phase_id doesn't exist"
-    return archive_name
-    
-def _determine_compiler_path(props):
-    # We need to segregate compiler builds based on both branch and builder
-    # TODO: better solution when branch is None
-    compiler_path = props['buildername']
-    if props.has_key('default_branch') and props['default_branch']:
-        compiler_path = props['default_branch']
-    elif props.has_key('branch') and props['branch']:
-        compiler_path = props['branch']
-    elif props.has_key('use_builder') and props['use_builder']:
-        compiler_path = props['use_builder']
-    return compiler_path
-
-def _determine_bootstrap_url(props):
-    if props.has_key('scheduler'):
-        name= ''
-        if props['scheduler'].startswith('phase2'):
-            # always use phase1 compiler for phase2
-            # TODO: this shouldn't be hard coded
-            name = 'clang-x86_64-darwin11-nobootstrap-RAincremental'
-        else:
-            # always use phase2 compiler for phase3 & phase4 compiler builds
-            # TODO: this shouldn't be hard coded
-            name = 'clang-x86_64-darwin11-RA'
-        curl = base_download_url + '/' + name + '/clang-' + props['phase_id']
-        curl += '.tar.gz'
-        return curl
-    else:
-        # if we get this far, we can assume that someone clicked 'rebuild'
-        # (otherwise it would have a scheduler, or not have a phase_id)
-        # we'll fall back to the phase1 build for this compiler
-        curl = base_download_url + '/clang-x86_64-darwin11-nobootstrap-RA/'
-        curl += props['buildername'] + '.tar.gz'
-        return curl
-
-def GetCompilerRoot(f):
-    # The following steps are used to retrieve a compiler archive
-    # clean out any existing archives
-    f.addStep(buildbot.steps.shell.ShellCommand(
-            name='rm.host-compiler',
-            command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'],
-            haltOnFailure=False, description=['rm', 'host-compiler'],
-            workdir=WithProperties('%(builddir)s')))
-    setProperty(f, 'rootURL', 
-                WithProperties( base_download_url + '/%(getpath)s/%(getname)s',
-                               getpath=_determine_compiler_path,
-                               getname=_determine_archive_name))
-    # curl down the archive
-    f.addStep(buildbot.steps.shell.ShellCommand(
-              name='download.artifacts',
-              command=['curl', curl_flags, 'host-compiler.tar.gz',
-                       WithProperties('%(rootURL)s')],
-              haltOnFailure=True,
-              description=['download build artifacts'],
-              workdir=WithProperties('%(builddir)s')))
-    # extract the compiler root from the archive
-    f.addStep(buildbot.steps.shell.ShellCommand(
-              name='unzip', command=['tar', '-zxvf','../host-compiler.tar.gz'],
-              haltOnFailure=True, description=['extract', 'host-compiler'],
-              workdir='host-compiler'))
-    return f
-
-def uploadArtifacts(f, rootdir='clang-install'):
-    #phase_id is required to make sure that path to archives are deterministic.
-    setProperty(f, 'phase_id', WithProperties('%(get_phase_id)s',
-                get_phase_id = determine_phase_id))
-    # we always create/name a compiler archive based on the same criteria
-    archive_path = WithProperties('%(builddir)s/%(getname)s',
-                                  getname=_determine_archive_name)
-    if rootdir.endswith('install'):
-        cit_path = 'clang-build/**/bin/c-index-test'
-        copy_command = 'cp %s %s/bin/' % (cit_path, rootdir)
-        f.addStep(buildbot.steps.shell.ShellCommand(
-                  name='add.cit', haltOnFailure=True,
-                  command = ['sh', '-c', copy_command],
-                  description=['add c-index-test to root'],
-                  workdir=WithProperties('%(builddir)s')))
-    f.addStep(buildbot.steps.shell.ShellCommand(
-              name='tar.and.zip', haltOnFailure=True,
-              command=['tar', 'czvf', archive_path, './'],
-              description=['tar', '&', 'zip'], workdir=rootdir))
-    # Upload the archive.
-    archive_dest = WithProperties(base_rsync_path +'/%(getpath)s/',
-                                  getpath=_determine_compiler_path)
-    f.addStep(buildbot.steps.shell.ShellCommand(
-              name='upload.artifacts', haltOnFailure=True,
-              command=['rsync', '-pave', 'ssh', archive_path, archive_dest],
-              description=['upload build artifacts'],
-              workdir=WithProperties('%(builddir)s')))
-    # Set the artifact URL in a property for easy access from the build log.
-    download_str = base_download_url + '/%(getpath)s/%(getname)s'
-    artifactsURL = WithProperties(download_str, getpath=_determine_compiler_path,
-                                  getname=_determine_archive_name)
-    setProperty(f, 'artifactsURL', artifactsURL)
-    return f
-
-def project_from_name(builder_name):
-  for project in ('apple-clang', 'clang'):
-      if project in builder_name:
-          return project
-  raise RuntimeError('Invalid builder name.')
-
-def determine_url(props):
-    if props.has_key('phase_id') and props.has_key('category'):
-        if props['category'].startswith('build-'):
-            return _determine_bootstrap_url(props)
-        project = project_from_name(props['buildername'])
-        name = props['use_builder']
-        curl = base_download_url + '/' + name + '/' + project_from_name(name)
-        curl += '-' + props['phase_id'] + '.tar.gz'
-        return curl
-    # phase_id does not exist, so this has to be a manually triggered build.
-    # we will fall back to the latest_validated build for the use_builder
-    # property if it exists, otherwise, fall back to the latest_validated build
-    # for this builder.
-    curl = base_download_url + '/latest_validated/'
-    if props.has_key('use_builder'):
-        curl += props['use_builder'] + '.tar.gz'
-    else:
-        curl += props['buildername'] + '.tar.gz'
-    return curl
-
-def GetCompilerArtifacts(f):
-    f.addStep(buildbot.steps.shell.ShellCommand(
-            name='rm.host-compiler',
-            command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'],
-            haltOnFailure=False, description=['rm', 'host-compiler'],
-            workdir=WithProperties('%(builddir)s')))
-    f.addStep(buildbot.steps.shell.ShellCommand(
-              name='download.artifacts',
-              command=['curl', curl_flags, 'host-compiler.tar.gz',
-                       WithProperties('%(get_curl)s', get_curl=determine_url)],
-              haltOnFailure=True, description=['download build artifacts'],
-              workdir=WithProperties('%(builddir)s')))
-    f.addStep(buildbot.steps.shell.ShellCommand(
-              name='unzip', command=['tar', '-zxvf','../host-compiler.tar.gz'],
-              haltOnFailure=True, description=['extract', 'host-compiler'],
-              workdir='host-compiler'))
-    return f
-
-def GetCCFromCompilerArtifacts(f, base_dir):
-    def get_cc(status, stdin, stdout):
-        lines = filter(bool, stdin.split('\n'))
-        for line in lines:
-            if 'bin/clang' in line:
-                cc_path = line
-                return { 'cc_path' : cc_path }
-        return { }
-    
-    f.addStep(buildbot.steps.shell.SetProperty(
-        name='find.cc',
-        command=['find', base_dir, '-name', 'clang'],
-        extract_fn=get_cc,
-        workdir=WithProperties('%(builddir)s')))
-    return f
-
-def GetCXXFromCompilerArtifacts(f, base_dir):
-    def get_cxx(status, stdin, stdout):
-        lines = filter(bool, stdin.split('\n'))
-        for line in lines:
-            if 'bin/clang++' in line:
-                cxx_path = line
-                return { 'cxx_path' : cxx_path }
-        return { }
-
-    f.addStep(buildbot.steps.shell.SetProperty(
-        name='find.cxx',
-        command=['find', base_dir, '-name', 'clang++'],
-        extract_fn=get_cxx,
-        workdir=WithProperties('%(builddir)s')))
-    return f
-

Modified: zorg/trunk/zorg/buildbot/PhasedBuilderUtils.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/PhasedBuilderUtils.py?rev=189650&r1=189649&r2=189650&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/PhasedBuilderUtils.py (original)
+++ zorg/trunk/zorg/buildbot/PhasedBuilderUtils.py Fri Aug 30 00:46:15 2013
@@ -118,12 +118,14 @@ def getUserDir(f):
     return f
 
 def GetLatestValidated(f):
+    import zorg.buildbot.util.artifacts as artifacts
+
     f.addStep(buildbot.steps.shell.ShellCommand(
             name='rm.host-compiler',
             command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'],
             haltOnFailure=False, description=['rm', 'host-compiler'],
             workdir=WithProperties('%(builddir)s')))
-    latest_url = zorg.buildbot.Artifacts.base_download_url
+    latest_url = artifacts.base_download_url
     latest_url += '/validated_builds/clang-x86_64-darwin11-R.tar.gz'
     f.addStep(buildbot.steps.shell.ShellCommand(
               name='download.artifacts',

Modified: zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/ClangBuilder.py?rev=189650&r1=189649&r2=189650&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/ClangBuilder.py (original)
+++ zorg/trunk/zorg/buildbot/builders/ClangBuilder.py Fri Aug 30 00:46:15 2013
@@ -8,7 +8,7 @@ from buildbot.steps.shell import Warning
 from buildbot.steps.source import SVN
 from buildbot.steps.transfer import FileDownload
 
-import zorg.buildbot.Artifacts as artifacts
+import zorg.buildbot.util.artifacts as artifacts
 import zorg.buildbot.builders.Util as builders_util
 import zorg.buildbot.PhasedBuilderUtils as phased_builder_utils
 import zorg.buildbot.commands as commands

Modified: zorg/trunk/zorg/buildbot/builders/LNTBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/LNTBuilder.py?rev=189650&r1=189649&r2=189650&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/LNTBuilder.py (original)
+++ zorg/trunk/zorg/buildbot/builders/LNTBuilder.py Fri Aug 30 00:46:15 2013
@@ -11,7 +11,7 @@ from buildbot.process.properties import
 import zorg
 from zorg.buildbot.builders import ClangBuilder
 from zorg.buildbot.PhasedBuilderUtils import getBuildDir, setProperty
-from zorg.buildbot.Artifacts import GetCompilerRoot, package_url
+from zorg.buildbot.util.artifacts import GetCompilerRoot, package_url
 
 def _get_cc(status, stdin, stdout):
     lines = filter(bool, stdin.split('\n'))

Modified: zorg/trunk/zorg/buildbot/builders/LibCXXBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/LibCXXBuilder.py?rev=189650&r1=189649&r2=189650&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/LibCXXBuilder.py (original)
+++ zorg/trunk/zorg/buildbot/builders/LibCXXBuilder.py Fri Aug 30 00:46:15 2013
@@ -9,7 +9,7 @@ import buildbot.steps.source.svn as svn
 import buildbot.process.properties as properties
 
 import zorg.buildbot.commands.LitTestCommand as lit_test_command
-import zorg.buildbot.Artifacts as artifacts
+import zorg.buildbot.util.artifacts as artifacts
 import zorg.buildbot.PhasedBuilderUtils as phased_builder_utils
 
 reload(lit_test_command)

Modified: zorg/trunk/zorg/buildbot/builders/gccSuiteBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/gccSuiteBuilder.py?rev=189650&r1=189649&r2=189650&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/gccSuiteBuilder.py (original)
+++ zorg/trunk/zorg/buildbot/builders/gccSuiteBuilder.py Fri Aug 30 00:46:15 2013
@@ -1,7 +1,7 @@
 import os
 import buildbot
 from zorg.buildbot.PhasedBuilderUtils import getBuildDir, setProperty
-from zorg.buildbot.Artifacts import GetCompilerArtifacts
+from zorg.buildbot.util.artifacts import GetCompilerArtifacts
 import zorg.buildbot.builders.ClangBuilder as ClangBuilder
 
 def gccRunSuite(languages):

Copied: zorg/trunk/zorg/buildbot/util/artifacts.py (from r189328, zorg/trunk/zorg/buildbot/Artifacts.py)
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/util/artifacts.py?p2=zorg/trunk/zorg/buildbot/util/artifacts.py&p1=zorg/trunk/zorg/buildbot/Artifacts.py&r1=189328&r2=189650&rev=189650&view=diff
==============================================================================
    (empty)





More information about the llvm-commits mailing list