[zorg] r261276 - Refactored sanitizer builder to use cmake and ninja.
Galina Kistanova via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 18 15:26:28 PST 2016
Author: gkistanova
Date: Thu Feb 18 17:26:27 2016
New Revision: 261276
URL: http://llvm.org/viewvc/llvm-project?rev=261276&view=rev
Log:
Refactored sanitizer builder to use cmake and ninja.
Modified:
zorg/trunk/zorg/buildbot/builders/SanitizerBuilderII.py
Modified: zorg/trunk/zorg/buildbot/builders/SanitizerBuilderII.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/SanitizerBuilderII.py?rev=261276&r1=261275&r2=261276&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/SanitizerBuilderII.py (original)
+++ zorg/trunk/zorg/buildbot/builders/SanitizerBuilderII.py Thu Feb 18 17:26:27 2016
@@ -1,69 +1,45 @@
-import os
-
-import buildbot
-import buildbot.process.factory
+from buildbot.process.factory import BuildFactory
+from buildbot.process.properties import WithProperties
from buildbot.steps.source import SVN
-from buildbot.steps.shell import ShellCommand
+from buildbot.steps.shell import ShellCommand, SetProperty
from buildbot.steps.shell import WarningCountingShellCommand
-from buildbot.process.properties import WithProperties
+from buildbot.steps.slave import RemoveDirectory
from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand
-
+from zorg.buildbot.commands.NinjaCommand import NinjaCommand
+from zorg.buildbot.conditions.FileConditions import FileDoesNotExist
def getSanitizerBuildFactoryII(
clean=False,
sanity_check=True,
sanitizers=['sanitizer','asan','lsan','msan','tsan','ubsan','dfsan'],
- build_type="Release",
- common_cmake_options=None,
- support_32_bit=True,
+ common_cmake_options=None, # FIXME: For backward compatibility. Will be removed.
+ extra_configure_args=[],
+ prefixCommand=["nice", "-n", "10"], # For backward compatibility.
env=None,
jobs="%(jobs)s",
timeout=1200):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
- llvm_objdir64 = "llvm64.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb', # Make sure Clang doesn't use color escape sequences.
}
- if env is not None:
+ if env:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
- f = buildbot.process.factory.BuildFactory()
+ f = BuildFactory()
- # Determine the build directory.
- f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir",
- command=["pwd"],
- property="builddir",
- description="set build dir",
- workdir=".",
- env=merged_env))
-
- # Clean up the source and build tree, if requested.
- if clean:
- f.addStep(ShellCommand(name="rm-llvm_llvm_srcdir",
- command=["rm", "-rf", llvm_srcdir],
- haltOnFailure=True,
- description=["rm src dir", "llvm"],
- workdir=".",
- env=merged_env))
-
- f.addStep(ShellCommand(name="rm-llvm_objdir",
- command=["rm", "-rf", llvm_objdir],
- haltOnFailure=True,
- description=["rm build dir", "llvm"],
- workdir=".",
- env=merged_env))
-
- f.addStep(ShellCommand(name="rm-llvm_objdir64",
- command=["rm", "-rf", llvm_objdir64],
- haltOnFailure=True,
- description=["rm build64 dir", "llvm"],
- workdir=".",
- env=merged_env))
+ # Clean directory, if requested.
+ cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
+ f.addStep(RemoveDirectory(name='clean '+llvm_objdir,
+ dir=llvm_objdir,
+ haltOnFailure=False,
+ flunkOnFailure=False,
+ doStepIf=cleanBuildRequested
+ ))
# Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind
f.addStep(SVN(name='svn-llvm',
@@ -114,135 +90,75 @@ def getSanitizerBuildFactoryII(
defaultBranch='trunk',
workdir='%s/projects/libunwind' % llvm_srcdir))
-
- lint_script = os.path.join("..", llvm_srcdir, "projects", "compiler-rt",
- "lib", "sanitizer_common", "scripts",
- "check_lint.sh")
-
# Run annotated command for sanitizer.
- f.addStep(AnnotatedCommand(name="lint",
- description="lint",
- timeout=timeout,
- haltOnFailure=True,
- command=lint_script,
- env=merged_env))
-
- # Create configuration files with cmake.
- f.addStep(ShellCommand(name="create-build-dir",
- command=["mkdir", "-p", llvm_objdir],
- haltOnFailure=True,
- description=["create build dir"],
- workdir=".",
- env=merged_env))
-
- # TODO: make it better way - use list for common_cmake_options and just merge.
- if common_cmake_options:
- cmakeCommand = [
- "cmake",
- "-DCMAKE_BUILD_TYPE=%s" % build_type,
- "-DCMAKE_C_COMPILER=clang",
- "-DCMAKE_CXX_COMPILER=clang++",
- "-DCMAKE_CXX_FLAGS='-std=c++11 -stdlib=libc++'",
- "%s" % common_cmake_options,
- "../%s" % llvm_srcdir]
- else:
- cmakeCommand = [
- "cmake",
- "-DCMAKE_BUILD_TYPE=%s" % build_type,
- "-DCMAKE_C_COMPILER=clang",
- "-DCMAKE_CXX_COMPILER=clang++",
- "-DCMAKE_CXX_FLAGS='-std=c++11 -stdlib=libc++'",
- "../%s" % llvm_srcdir]
+ if sanity_check:
+ f.addStep(
+ AnnotatedCommand(
+ name="lint check",
+ description="lint check",
+ timeout=timeout,
+ haltOnFailure=False, #True,
+ warnOnWarnings=True,
+ command=["./check_lint.sh"],
+ workdir="%s/projects/compiler-rt/lib/sanitizer_common/scripts" % llvm_srcdir,
+ env=merged_env))
+
+ # Always build with ninja.
+ cmakeCommand = ["cmake", "-G", "Ninja"]
+
+ # Reconsile configure args with the defaults we want.
+ if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
+ cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release')
+ if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args):
+ cmakeCommand.append('-DLLVM_ENABLE_WERROR=OFF')
+ if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args):
+ cmakeCommand.append('-DLLVM_ENABLE_ASSERTIONS=ON')
+ if not any(a.startswith('-DCMAKE_C_COMPILER') for a in extra_configure_args):
+ cmakeCommand.append('-DCMAKE_C_COMPILER=clang')
+ if not any(a.startswith('-DCMAKE_CXX_COMPILER') for a in extra_configure_args):
+ cmakeCommand.append('-DCMAKE_CXX_COMPILER=clang++')
+ if not any(a.startswith('-DCMAKE_CXX_FLAGS') for a in extra_configure_args):
+ cmakeCommand.append('-DCMAKE_CXX_FLAGS=\"-std=c++11 -stdlib=libc++\"')
+ if not any(a.startswith('-DCMAKE_EXE_LINKER_FLAGS') for a in extra_configure_args):
+ cmakeCommand.append('-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt')
+ if not any(a.startswith('-DLIBCXXABI_USE_LLVM_UNWINDER=') for a in extra_configure_args):
+ cmakeCommand.append('-DLIBCXXABI_USE_LLVM_UNWINDER=ON')
+ if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
+ cmakeCommand.append('-DLLVM_LIT_ARGS=\"-v\"')
+
+ cmakeCommand += extra_configure_args + ["../%s" % llvm_srcdir]
- # Note: ShellCommand does not pass the params with special symbols correctly.
+ # Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
- f.addStep(ShellCommand(name="cmake-configure-1",
- description=["cmake configure, phase 1"],
- haltOnFailure=True,
- command=WithProperties(" ".join(cmakeCommand)),
- workdir=llvm_objdir,
- env=merged_env))
+ f.addStep(ShellCommand(
+ name="cmake-configure",
+ description=["cmake configure"],
+ haltOnFailure=False, #True,
+ warnOnWarnings=True,
+ command=WithProperties(" ".join(cmakeCommand)),
+ env=merged_env,
+ workdir=llvm_objdir,
+ doStepIf=FileDoesNotExist("./%s/CMakeCache.txt" % llvm_objdir)))
# Build everything.
- f.addStep(WarningCountingShellCommand(name="compile",
- command=['nice', '-n', '10',
- 'make', WithProperties("-j%s" % jobs)],
- haltOnFailure=True,
- description=["compiling"],
- descriptionDone=["compile"],
- workdir=llvm_objdir,
- env=merged_env))
+ f.addStep(NinjaCommand(name='build',
+ haltOnFailure=False, #True,
+ warnOnWarnings=True,
+ description=['building', 'with', 'ninja'],
+ descriptionDone=['built', 'with', 'ninja'],
+ workdir=llvm_objdir,
+ env=merged_env))
# Run tests for each of the requested sanitizers.
- if sanity_check and sanitizers:
- for item in sanitizers:
- f.addStep(WarningCountingShellCommand(name="make-check-%s"% item,
- command=['make', 'check-%s' % item,
- WithProperties("-j%s" % jobs)],
- haltOnFailure=False,
- description=["make check-%s" % item,],
- descriptionDone=["make check-%s" % item,],
- workdir=llvm_objdir,
- env=merged_env))
-
- # build 64-bit llvm using clang
-
- # Create configuration files with cmake.
- f.addStep(ShellCommand(name="create-llvm_build64-dir",
- command=["mkdir", "-p", llvm_objdir64],
- haltOnFailure=True,
- description=["create llvm_build64 dir"],
- workdir=".",
- env=merged_env))
-
- # Use just built compiler.
- clang_path = "%(builddir)s" + "/%s/bin" % llvm_objdir
-
- if common_cmake_options:
- cmakeCommand_llvm64 = [
- "cmake",
- "-DCMAKE_BUILD_TYPE=%s" % build_type,
- "%s" % common_cmake_options,
- "-DCMAKE_C_COMPILER=%s/clang" % clang_path,
- "-DCMAKE_CXX_COMPILER=%s/clang++" % clang_path,
- "-DCMAKE_CXX_FLAGS='-I" + "%(builddir)s" + "/%s/projects/libcxx/include -std=c++11 -stdlib=libc++'" % llvm_srcdir,
- "../%s" % llvm_srcdir]
- else:
- cmakeCommand_llvm64 = [
- "cmake",
- "-DCMAKE_BUILD_TYPE=%s" % build_type,
- "-DCMAKE_C_COMPILER=%s/clang" % clang_path,
- "-DCMAKE_CXX_COMPILER=%s/clang++" % clang_path,
- "-DCMAKE_CXX_FLAGS='-I" + "%(builddir)s" + "/%s/projects/libcxx/include -std=c++11 -stdlib=libc++'" % llvm_srcdir,
- "../%s" % llvm_srcdir]
+ if sanitizers:
+ for s in sanitizers:
+ f.addStep(
+ NinjaCommand(name='test %s' % s,
+ targets=['check-%s' % s],
+ haltOnFailure=False, #True,
+ description=['testing', '%s' % s],
+ descriptionDone=['test', '%s' % s],
+ workdir=llvm_objdir,
+ env=merged_env))
- # Note: ShellCommand does not pass the params with special symbols correctly.
- # The " ".join is a workaround for this bug.
- f.addStep(ShellCommand(name="cmake-configure-2",
- description=["cmake configure 64-bit llvm"],
- haltOnFailure=True,
- command=WithProperties(" ".join(cmakeCommand_llvm64)),
- workdir=llvm_objdir64,
- env=merged_env))
-
- f.addStep(WarningCountingShellCommand(name="compile 64-bit llvm",
- command=['nice', '-n', '10',
- 'make', WithProperties("-j%s" % jobs)],
- haltOnFailure=True,
- description=["compiling"],
- descriptionDone=["compile"],
- workdir=llvm_objdir64,
- env=merged_env))
-
- # Run tests for each of the requested sanitizers.
- if sanity_check and sanitizers:
- for item in sanitizers:
- f.addStep(WarningCountingShellCommand(name="make-check-%s 64-bit"% item,
- command=['make', 'check-%s' % item,
- WithProperties("-j%s" % jobs)],
- haltOnFailure=False,
- description=["make check-%s 64-bit" % item,],
- descriptionDone=["make check-%s" % item,],
- workdir=llvm_objdir64,
- env=merged_env))
return f
More information about the llvm-commits
mailing list