[zorg] r290270 - [buildbot] Added CUDATestsuiteBuilder.py
Artem Belevich via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 21 13:15:57 PST 2016
Author: tra
Date: Wed Dec 21 15:15:56 2016
New Revision: 290270
URL: http://llvm.org/viewvc/llvm-project?rev=290270&view=rev
Log:
[buildbot] Added CUDATestsuiteBuilder.py
getCUDATestsuiteBuildFactory() returns a factory that:
* checks out clang, libcxx(+deps), and test-suite.
* compiles & tests clang.
* compiles and runs CUDA tests.
Added checkout_libcxx and checkout_test_suite knobs to
getClangCmakeBuildFactory() so it can be reused for other
clang-based buildbots.
Differential Revision: https://reviews.llvm.org/D27766
Added:
zorg/trunk/zorg/buildbot/builders/CUDATestsuiteBuilder.py
Modified:
zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
Added: zorg/trunk/zorg/buildbot/builders/CUDATestsuiteBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/CUDATestsuiteBuilder.py?rev=290270&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/CUDATestsuiteBuilder.py (added)
+++ zorg/trunk/zorg/buildbot/builders/CUDATestsuiteBuilder.py Wed Dec 21 15:15:56 2016
@@ -0,0 +1,186 @@
+from buildbot.process.properties import WithProperties
+try:
+ # buildbot 0.8.5
+ from buildbot.steps.slave import RemoveDirectory
+except:
+ # buildbot 0.8.12
+ from buildbot.plugins import steps
+ RemoveDirectory = steps.RemoveDirectory
+
+from zorg.buildbot.builders import ClangBuilder
+from zorg.buildbot.commands.CmakeCommand import CmakeCommand
+from zorg.buildbot.commands.NinjaCommand import NinjaCommand
+
+
+def getCUDATestsuiteBuildFactory(
+ externals, # Directory with CUDA, thrust and gcc versions for testing.
+ always_clean=True,
+ test=False,
+ useTwoStage=False,
+ cmake='cmake',
+ extra_cmake_args=None, # Extra CMake args for all stages.
+ extra_ts_cmake_args=None, # extra cmake args for testsuite.
+ jobs=None,
+ cuda_jobs=1, # number of simultaneous CUDA apps to run
+ env=None, # Environmental variables for all steps.
+ enable_thrust_tests=False,
+ split_thrust_tests=False, # Each thrust test is a separate executable.
+ run_thrust_tests=False,
+ enable_libcxx=True, # checkout/build libcxx to test with.
+ gpu_arch_list=None,
+ gpu_devices=None, # List of devices to make visible to CUDA
+ stage1_config='Release',
+ stage2_config='Release'):
+
+ if extra_cmake_args is None:
+ extra_cmake_args = []
+ if extra_ts_cmake_args is None:
+ extra_ts_cmake_args = []
+
+ # Prepare environmental variables. Set here all env we want for all steps.
+ merged_env = {
+ 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
+ }
+ if env is not None:
+ # Overwrite pre-set items with the given ones, so user can set
+ # anything.
+ merged_env.update(env)
+
+ source_dir = 'llvm' # Should match the one used in getClangCMakeBuildFactory.
+ stage1_build_dir = 'stage1' # Should match the one defined in getClangCMakeBuildFactory.
+ stage2_build_dir = 'stage2' # Should match the one defined in getClangCMakeBuildFactory.
+
+ if useTwoStage:
+ clang_build_dir = stage2_build_dir
+ else:
+ clang_build_dir = stage1_build_dir
+
+ # Build clang.
+ f = ClangBuilder.getClangCMakeBuildFactory(
+ clean=always_clean,
+ test=test,
+ cmake=cmake,
+ extra_cmake_args=extra_cmake_args,
+ jobs=jobs,
+ env=merged_env,
+ useTwoStage=useTwoStage,
+ stage1_config=stage1_config,
+ stage2_config=stage2_config,
+ checkout_libcxx=enable_libcxx,
+ checkout_test_suite=True)
+
+ cuda_test_env = {
+ 'PYTHONPATH': WithProperties("%(workdir)s/" + source_dir +
+ "/utils/lit:${PYTHONPATH}"),
+ 'PATH': WithProperties("%(workdir)s/" + clang_build_dir +
+ "/bin:${PATH}"),
+ }
+ # Limit GPUs visible to CUDA.
+ if gpu_devices:
+ cuda_test_env["CUDA_VISIBLE_DEVICES"] = ",".join(
+ str(x) for x in gpu_devices)
+ merged_env.update(cuda_test_env)
+
+ ts_build_dir = 'test-suite-build'
+
+ # Completely remove test suite build dir.
+ f.addStep(
+ RemoveDirectory(name="Remove old test-suite build directory",
+ dir=ts_build_dir))
+
+ if extra_ts_cmake_args:
+ cmake_args = extra_ts_cmake_args[:]
+ else:
+ cmake_args = []
+
+ # Set proper defaults.
+ CmakeCommand.applyDefaultOptions(cmake_args, [
+ ('-DCMAKE_BUILD_TYPE=', 'Release'),
+ ])
+
+ # Some options are required for this stage no matter what.
+ CmakeCommand.applyRequiredOptions(cmake_args, [
+ ('-G', 'Ninja'),
+ ])
+
+ cmake_args.append(
+ WithProperties(
+ "-DCMAKE_CXX_COMPILER=%(workdir)s/" +
+ clang_build_dir + "/bin/clang++"
+ ))
+ cmake_args.append(
+ WithProperties(
+ "-DCMAKE_C_COMPILER=%(workdir)s/" + clang_build_dir + "/bin/clang"
+ ))
+
+ cmake_args.append('-DTEST_SUITE_SUBDIRS=External'),
+ # Limit to External tests only.
+
+ if externals:
+ cmake_args.append('-DTEST_SUITE_EXTERNALS_DIR=' + externals)
+ if split_thrust_tests:
+ cmake_args.append('-DTHRUST_SPLIT_TESTS=1')
+ if gpu_arch_list:
+ cmake_args.append('-DCUDA_GPU_ARCH=' + ';'.join(gpu_arch_list))
+ if cuda_jobs:
+ cmake_args.append('-DCUDA_JOBS=%s' % cuda_jobs)
+
+ # Then do fresh cmake configuration.
+ f.addStep(CmakeCommand(name='cmake test-suite',
+ description='cmake test-suite',
+ haltOnFailure=True,
+ options=cmake_args,
+ path="../test/test-suite",
+ workdir=ts_build_dir,
+ env=merged_env))
+
+ # Always build simple CUDA tests. They serve as compilation
+ # smoketests and will fail quickly if compiler has obvious issues
+ # compiling CUDA files.
+ f.addStep(NinjaCommand(
+ name='ninja build simple CUDA tests',
+ targets=["cuda-tests-simple"],
+ jobs=jobs,
+ haltOnFailure=True,
+ description=["building simple CUDA tests"],
+ descriptionDone=["simple CUDA tests built."],
+ workdir=ts_build_dir,
+ env=merged_env))
+
+ f.addStep(NinjaCommand(
+ name='run simple CUDA tests',
+ targets=["check-cuda-simple"],
+ jobs=1, # lit will parallelize the jobs
+ haltOnFailure=True,
+ description=["Running simple CUDA tests"],
+ descriptionDone=["simple CUDA tests done."],
+ workdir=ts_build_dir,
+ env=merged_env))
+
+ # If we've enabled thrust tests, build them now.
+ # WARNING: This takes a lot of time to build.
+ if (enable_thrust_tests):
+ f.addStep(NinjaCommand(
+ name='ninja build thrust',
+ targets=["cuda-tests-thrust"],
+ jobs=jobs,
+ haltOnFailure=True,
+ description=["building thrust tests"],
+ descriptionDone=["thrust tests built."],
+ workdir=ts_build_dir,
+ env=merged_env))
+ # Run them. That also takes a while.
+ # TODO(tra) we may want to run more than one instance so one
+ # can be compiling tests while another is running them on GPU.
+ if run_thrust_tests:
+ f.addStep(NinjaCommand(
+ name='run all CUDA tests',
+ targets=["check"],
+ jobs=1, # lit will parallelize the jobs.
+ haltOnFailure=True,
+ description=["running all CUDA tests."],
+ descriptionDone=["all cuda tests done."],
+ workdir=ts_build_dir,
+ env=merged_env))
+
+ return f
Modified: zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/ClangBuilder.py?rev=290270&r1=290269&r2=290270&view=diff
==============================================================================
--- zorg/trunk/zorg/buildbot/builders/ClangBuilder.py (original)
+++ zorg/trunk/zorg/buildbot/builders/ClangBuilder.py Wed Dec 21 15:15:56 2016
@@ -340,7 +340,8 @@ def addSVNUpdateSteps(f,
checkout_clang_tools_extra,
checkout_compiler_rt,
checkout_test_suite,
- checkout_lld):
+ checkout_lld,
+ checkout_libcxx):
# We *must* checkout at least Clang+LLVM
f.addStep(SVN(name='svn-llvm',
mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
@@ -376,6 +377,22 @@ def addSVNUpdateSteps(f,
mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir='llvm/tools/lld'))
+ if checkout_libcxx:
+ f.addStep(SVN(name='svn-libcxx',
+ mode='update',
+ baseURL='http://llvm.org/svn/llvm-project/libcxx/',
+ defaultBranch='trunk',
+ workdir='llvm/projects/libcxx'))
+ f.addStep(SVN(name='svn-libcxxabi',
+ mode='update',
+ baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
+ defaultBranch='trunk',
+ workdir='llvm/projects/libcxxabi'))
+ f.addStep(SVN(name='svn-libunwind',
+ mode='update',
+ baseURL='http://llvm.org/svn/llvm-project/libunwind/',
+ defaultBranch='trunk',
+ workdir='llvm/projects/libunwind'))
def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env,
gcs_url_property=None):
@@ -503,7 +520,9 @@ def getClangCMakeBuildFactory(
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
- checkout_lld=True):
+ checkout_lld=True,
+ checkout_libcxx=False,
+ checkout_test_suite=False):
return _getClangCMakeBuildFactory(
clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
@@ -513,7 +532,9 @@ def getClangCMakeBuildFactory(
env=env, extra_cmake_args=extra_cmake_args,
checkout_clang_tools_extra=checkout_clang_tools_extra,
checkout_lld=checkout_lld,
- checkout_compiler_rt=checkout_compiler_rt)
+ checkout_compiler_rt=checkout_compiler_rt,
+ checkout_libcxx=checkout_libcxx,
+ checkout_test_suite=checkout_test_suite)
def _getClangCMakeBuildFactory(
clean=True,
@@ -546,6 +567,8 @@ def _getClangCMakeBuildFactory(
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
+ checkout_libcxx=False,
+ checkout_test_suite=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
@@ -560,7 +583,8 @@ def _getClangCMakeBuildFactory(
checkout_clang_tools_extra=checkout_clang_tools_extra,
checkout_compiler_rt=checkout_compiler_rt,
checkout_lld=checkout_lld,
- checkout_test_suite=runTestSuite)
+ checkout_test_suite=runTestSuite or checkout_test_suite,
+ checkout_libcxx=checkout_libcxx)
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
@@ -966,4 +990,3 @@ def getClangTestsIgnoresFromPath(path, k
ignores['gdb-1472-testsuite' ] = gdb_dg_ignores
return ignores
-
More information about the llvm-commits
mailing list