[zorg] r352569 - Switch sanitizer-windows bot to annotator scripts
Reid Kleckner via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 29 15:47:34 PST 2019
Author: rnk
Date: Tue Jan 29 15:47:34 2019
New Revision: 352569
URL: http://llvm.org/viewvc/llvm-project?rev=352569&view=rev
Log:
Switch sanitizer-windows bot to annotator scripts
Summary:
This is intended to be equivalent to the old SanitizerBuilderWindows
buidlbot factory code. Next, I'll look into enabling libfuzzer testing
on this bot, and make sure it's on VS 2017.
Reviewers: gkistanova, inglorion, jfb, serge-sans-paille
Subscribers: llvm-commits
Differential Revision: https://reviews.llvm.org/D57414
Added:
zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py
Removed:
zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py
Modified:
zorg/trunk/buildbot/osuosl/master/config/builders.py
zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py
Modified: zorg/trunk/buildbot/osuosl/master/config/builders.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/builders.py?rev=352569&r1=352568&r2=352569&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/builders.py (original)
+++ zorg/trunk/buildbot/osuosl/master/config/builders.py Tue Jan 29 15:47:34 2019
@@ -7,7 +7,6 @@ from zorg.buildbot.builders import LLGoB
from zorg.buildbot.builders import ClangAndLLDBuilder
from zorg.buildbot.builders import SanitizerBuilder
from zorg.buildbot.builders import SanitizerBuilderII
-from zorg.buildbot.builders import SanitizerBuilderWindows
from zorg.buildbot.builders import OpenMPBuilder
from zorg.buildbot.builders import LibcxxAndAbiBuilder
from zorg.buildbot.builders import SphinxDocsBuilder
@@ -1119,8 +1118,9 @@ def _get_sanitizer_builders():
{'name': "sanitizer-windows",
'slavenames' :["sanitizer-windows"],
'builddir': "sanitizer-windows",
- 'factory': SanitizerBuilderWindows.getSanitizerWindowsBuildFactory(
- vs='%VS140COMNTOOLS%')},
+ 'factory': AnnotatedBuilder.getAnnotatedBuildFactory(
+ script="sanitizer-windows.py",
+ depends_on_projects=["llvm", "clang", "lld", "compiler-rt"])},
## ARMv7 check-all full (compiler-rt) with CMake builder; Needs x86 for ASAN tests
{'name': "clang-cmake-armv7-full",
Removed: zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py?rev=352568&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py (original)
+++ zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py (removed)
@@ -1,117 +0,0 @@
-import os
-
-import buildbot
-import buildbot.process.factory
-from buildbot.steps.shell import SetProperty
-from buildbot.steps.shell import ShellCommand
-from buildbot.steps.slave import RemoveDirectory
-from buildbot.steps.source import SVN
-from buildbot.process.properties import Property
-from buildbot.process.properties import WithProperties
-from zorg.buildbot.builders.Util import getVisualStudioEnvironment
-from zorg.buildbot.builders.Util import extractSlaveEnvironment
-from zorg.buildbot.builders.Util import extractClangVersion
-from zorg.buildbot.commands.NinjaCommand import NinjaCommand
-
-def getSource(f,llvmTopDir='llvm'):
- f.addStep(SVN(name='svn-llvm',
- mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
- defaultBranch='trunk',
- workdir=llvmTopDir))
- f.addStep(SVN(name='svn-clang',
- mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/',
- defaultBranch='trunk',
- workdir='%s/tools/clang' % llvmTopDir))
- f.addStep(SVN(name='svn-lld',
- mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/',
- defaultBranch='trunk',
- workdir='%s/tools/lld' % llvmTopDir))
- f.addStep(SVN(name='svn-compiler-rt',
- mode='update',
- baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
- defaultBranch='trunk',
- workdir='%s/projects/compiler-rt' % llvmTopDir))
- return f
-
-def getSanitizerWindowsBuildFactory(
- clean=False,
- cmake='cmake',
-
- # Default values for VS devenv and build configuration
- vs=r"""%VS120COMNTOOLS%""",
- config='Release',
- target_arch='x86',
-
- extra_cmake_args=[]):
-
- ############# PREPARING
- f = buildbot.process.factory.BuildFactory()
-
- # Kill any stale symbolizer processes for the last run. If there are any
- # stale processes, the build will fail during linking. This can happen to
- # any process, but it is most likely to happen to llvm-symbolizer if its
- # pipe isn't closed.
- taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0'
- f.addStep(ShellCommand(name='taskkill',
- description='kill stale processes',
- command=['cmd', '/c', taskkill_cmd],
- haltOnFailure=False))
-
- # Determine Slave Environment and Set MSVC environment.
- f.addStep(SetProperty(
- command=getVisualStudioEnvironment(vs, target_arch),
- extract_fn=extractSlaveEnvironment))
-
- f = getSource(f,'llvm')
-
- # Global configurations
- build_dir = 'build'
-
- ############# CLEANING
- cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
- f.addStep(RemoveDirectory(name='clean '+build_dir,
- dir=build_dir,
- haltOnFailure=False,
- flunkOnFailure=False,
- doStepIf=cleanBuildRequested
- ))
-
- f.addStep(ShellCommand(name='cmake',
- command=[cmake, "-G", "Ninja", "../llvm",
- "-DCMAKE_BUILD_TYPE="+config,
- "-DLLVM_ENABLE_ASSERTIONS=ON"]
- + extra_cmake_args,
- haltOnFailure=True,
- workdir=build_dir,
- env=Property('slave_env')))
-
- # Build compiler-rt first to speed up detection of Windows-specific
- # compiler-time errors in the sanitizer runtimes.
- f.addStep(NinjaCommand(name='build compiler-rt',
- targets=['compiler-rt'],
- haltOnFailure=True,
- description='ninja compiler-rt',
- workdir=build_dir,
- env=Property('slave_env')))
-
- # Build Clang and LLD next so that most compilation errors occur in a build
- # step.
- f.addStep(NinjaCommand(name='build clang lld',
- targets=['clang', 'lld'],
- haltOnFailure=True,
- description='ninja clang lld',
- workdir=build_dir,
- env=Property('slave_env')))
-
- # Only run sanitizer tests.
- # Don't build targets that are not required in order to speed up the cycle.
- for target in ['check-asan', 'check-asan-dynamic', 'check-sanitizer',
- 'check-cfi']:
- f.addStep(NinjaCommand(name='run %s' % target,
- targets=[ target ],
- haltOnFailure=False,
- description='ninja %s' % target,
- workdir=build_dir,
- env=Property('slave_env')))
-
- return f
Modified: zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py?rev=352569&r1=352568&r2=352569&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py (original)
+++ zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py Tue Jan 29 15:47:34 2019
@@ -231,12 +231,29 @@ class AnnotatedBuilder:
def update_sources(self, source_dir, projects, revision, svn='svn'):
self.report_build_step('update-sources')
self.halt_on_failure()
+
+ # TODO: This needs to be updated to use the monorepo.
+ # Where to check the project out relative to an LLVM checkout.
+ checkout_locations = {
+ 'llvm': '',
+ 'clang': 'tools/clang',
+ 'lld': 'tools/lld',
+ 'compiler-rt': 'projects/compiler-rt',
+ }
+ # If the project is named differently in svn, put it here.
+ svn_locations = { 'clang': 'cfe' }
+ svn_uri_pattern = 'https://llvm.org/svn/llvm-project/%s/trunk'
+
try:
- for (project, path, uri) in projects:
- if path is None:
+ for project in projects:
+ # TODO: Fail the build and report an error if we don't know the
+ # checkout location.
+ path = checkout_locations[project]
+ if not path:
path = source_dir
elif not os.path.isabs(path):
path = pjoin(source_dir, path)
+ uri = svn_uri_pattern % (svn_locations.get(project, project),)
util.report("Updating %s to %s at %s from %s" %
(project, revision, util.shquote(path), uri))
if os.path.exists(pjoin(path, '.svn')):
@@ -252,6 +269,7 @@ class AnnotatedBuilder:
def run_steps(
self,
stages=1,
+ projects=None,
check_targets=None,
check_stages=None,
extra_cmake_args=None,
@@ -263,6 +281,8 @@ class AnnotatedBuilder:
jobs=None):
"""
stages: number of stages to run (default: 1)
+ projects: which subprojects to check out from SVN
+ llvm must be first in the list (default: ['llvm', 'clang', 'lld'])
check_targets: targets to run during the check phase (default: ['check-all'])
check_stages: stages for which to run the check phase
(array of bool, default: all True)
@@ -290,6 +310,8 @@ class AnnotatedBuilder:
revision = os.environ.get('BUILDBOT_REVISION')
if stage1_extra_cmake_args is None:
stage1_extra_cmake_args = extra_cmake_args
+ if projects is None:
+ projects = ['llvm', 'clang', 'lld']
c_compiler, cxx_compiler = self.compiler_binaries(compiler)
@@ -323,14 +345,7 @@ class AnnotatedBuilder:
cwd = os.getcwd()
source_dir = pjoin(cwd, 'llvm.src')
build_dir = pjoin(cwd, 'build')
- svn_uri_pattern = 'https://llvm.org/svn/llvm-project/%s/trunk'
- projects = []
- projects.append(('llvm', None, svn_uri_pattern % ('llvm',)))
- projects.append(
- ('clang', pjoin('tools', 'clang'), svn_uri_pattern % ('cfe',)))
cmake_args = ['-GNinja']
- for p in ['lld']:
- projects.append((p, pjoin('tools', p), svn_uri_pattern % (p,)))
self.update_sources(source_dir, projects, revision)
@@ -363,9 +378,9 @@ def get_vcvars(vs_tools, arch):
if not arch:
# First check the wow64 processor architecture, since python is probably
# 32-bit, then fall back to PROCESSOR_ARCHITECTURE.
- arch = os.environ['PROCESSOR_ARCHITEW6432'].lower()
+ arch = os.environ.get('PROCESSOR_ARCHITEW6432', '').lower()
if not arch:
- arch = os.environ['PROCESSOR_ARCHITECTURE'].lower()
+ arch = os.environ.get('PROCESSOR_ARCHITECTURE', '').lower()
else:
arch = arch.lower()
Added: zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py?rev=352569&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py (added)
+++ zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py Tue Jan 29 15:47:34 2019
@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+import os
+import sys
+import annotated_builder
+import util
+
+
+class SanitizerAnnotatedBuilder(annotated_builder.AnnotatedBuilder):
+
+ """Customizes the 'build' step of the generic AnnotatedBuilder"""
+
+ def build(self, stage_name, build_dir, jobs=None):
+ # The basic idea here is to run 'ninja compiler-rt ; ninja clang lld'.
+ # This ensures that portability issues in compiler-rt code are found
+ # first. Then, we only build clang and lld, the primary dependencies of
+ # the sanitizer test suites, to keep cycle time low. This means there
+ # are still some remaining test dependencies (FileCheck) that may be
+ # compiled during the check step, but there shouldn't be that many.
+ self.report_build_step('%s build' % (stage_name,))
+ self.halt_on_failure()
+ base_cmd = ['ninja']
+ if jobs:
+ base_cmd += ['-j', str(jobs)]
+ early_targets = ['compiler-rt']
+ late_targets = ['clang', 'lld']
+ util.report_run_cmd(base_cmd + early_targets, cwd=build_dir)
+ util.report_run_cmd(base_cmd + late_targets, cwd=build_dir)
+
+
+def main(argv):
+ ap = annotated_builder.get_argument_parser()
+ args = ap.parse_args(argv[1:])
+
+ projects = ['llvm', 'clang', 'lld', 'compiler-rt']
+ stages = 1
+ stage1_extra_cmake_args = [
+ '-DCMAKE_BUILD_TYPE=Release',
+ '-DLLVM_ENABLE_PDB=ON',
+ '-DLLVM_ENABLE_ASSERTIONS=ON',
+ '-DLLVM_TARGETS_TO_BUILD=X86',
+ ]
+ extra_cmake_args = stage1_extra_cmake_args + [
+ '-DLLVM_USE_LINKER=lld',
+ ]
+ check_targets = ['check-asan', 'check-asan-dynamic', 'check-sanitizer',
+ 'check-cfi']
+
+ # These arguments are a bit misleading, they really mean use cl.exe for
+ # stage1 instead of GCC.
+ compiler = 'clang-cl'
+ linker = 'lld-link'
+
+ builder = SanitizerAnnotatedBuilder()
+ builder.run_steps(stages=stages,
+ projects=projects,
+ extra_cmake_args=extra_cmake_args,
+ stage1_extra_cmake_args=stage1_extra_cmake_args,
+ check_targets=check_targets,
+ compiler=compiler,
+ linker=linker,
+ jobs=args.jobs)
+
+
+if __name__ == '__main__':
+ sys.path.append(os.path.dirname(__file__))
+ sys.exit(main(sys.argv))
More information about the llvm-commits
mailing list