[zorg] r280435 - First round of re-factoring to use LLVMBuildFactory for automatic scheduling.
Renato Golin via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 6 03:26:13 PDT 2016
Hi Galina,
This commit (and all related refactoring changes) have completely
broken our buildbots to a point where we can't build or work around,
notably:
* The sources are checked out in llvm.src but the build wants llvm
* The lnt and test-suite directories are now checked in llvm.src/test,
which is completely the wrong place
* Check-all fails because of lit problems understanding lnt/test-suite
This refactoring wasn't announced, we haven'd had time to test or
validate, and now everything is broken. Can you please revert these
changes and restart the master ASAP?
Next steps are to make that changes locally and we should validate on
our builders *before* it goes in production.
thanks,
--renato
On 2 September 2016 at 01:16, Galina Kistanova via llvm-commits
<llvm-commits at lists.llvm.org> wrote:
> Author: gkistanova
> Date: Thu Sep 1 19:16:40 2016
> New Revision: 280435
>
> URL: http://llvm.org/viewvc/llvm-project?rev=280435&view=rev
> Log:
> First round of re-factoring to use LLVMBuildFactory for automatic scheduling.
>
> Modified:
> zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
>
> Modified: zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
> URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/ClangBuilder.py?rev=280435&r1=280434&r2=280435&view=diff
> ==============================================================================
> --- zorg/trunk/zorg/buildbot/builders/ClangBuilder.py (original)
> +++ zorg/trunk/zorg/buildbot/builders/ClangBuilder.py Thu Sep 1 19:16:40 2016
> @@ -8,8 +8,11 @@ from buildbot.process.properties import
> from buildbot.steps.shell import Configure, ShellCommand, SetProperty
> from buildbot.steps.shell import WarningCountingShellCommand
> from buildbot.steps.source import SVN
> +from buildbot.steps.slave import RemoveDirectory
> from buildbot.steps.transfer import FileDownload
>
> +from zorg.buildbot.process.factory import LLVMBuildFactory
> +
> import zorg.buildbot.util.artifacts as artifacts
> import zorg.buildbot.builders.Util as builders_util
> import zorg.buildbot.util.phasedbuilderutils as phasedbuilderutils
> @@ -17,6 +20,7 @@ import zorg.buildbot.commands as command
> import zorg.buildbot.commands.BatchFileDownload as batch_file_download
> import zorg.buildbot.commands.LitTestCommand as lit_test_command
>
> +
> def getClangBuildFactory(
> triple=None,
> clean=True,
> @@ -335,40 +339,6 @@ def getClangBuildFactory(
>
> return f
>
> -def addSVNUpdateSteps(f,
> - checkout_clang_tools_extra,
> - checkout_compiler_rt,
> - checkout_test_suite):
> - # We *must* checkout at least Clang+LLVM
> - f.addStep(SVN(name='svn-llvm',
> - mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
> - defaultBranch='trunk',
> - workdir='llvm'))
> - f.addStep(SVN(name='svn-clang',
> - mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/',
> - defaultBranch='trunk',
> - workdir='llvm/tools/clang'))
> -
> - # Extra stuff that will be built/tested
> - if checkout_clang_tools_extra:
> - f.addStep(SVN(name='svn-clang-tools-extra',
> - mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
> - defaultBranch='trunk',
> - workdir='llvm/tools/clang/tools/extra'))
> - if checkout_compiler_rt:
> - f.addStep(SVN(name='svn-compiler-rt',
> - mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
> - defaultBranch='trunk',
> - workdir='llvm/projects/compiler-rt'))
> - if checkout_test_suite:
> - f.addStep(SVN(name='svn-lnt',
> - mode='update', baseURL='http://llvm.org/svn/llvm-project/lnt/',
> - defaultBranch='trunk',
> - workdir='test/lnt'))
> - f.addStep(SVN(name='svn-test-suite',
> - mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/',
> - defaultBranch='trunk',
> - workdir='test/test-suite'))
>
> def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env,
> gcs_url_property=None):
> @@ -416,32 +386,33 @@ def addGCSUploadSteps(f, package_name, i
> workdir=install_prefix,
> env=env))
>
> +
> def getClangCMakeGCSBuildFactory(
> clean=True,
> test=True,
> - cmake='cmake',
> + cmake=None,
> jobs=None,
>
> # VS tools environment variable if using MSVC. For example,
> # %VS120COMNTOOLS% selects the 2013 toolchain.
> vs=None,
> - vs_target_arch='x86',
> + vs_target_arch=None,
>
> # Multi-stage compilation
> useTwoStage=False,
> testStage1=True,
> - stage1_config='Release',
> - stage2_config='Release',
> + stage1_config=None,
> + stage2_config=None,
>
> # Test-suite
> runTestSuite=False,
> - nt_flags=[],
> + nt_flags=None,
> submitURL=None,
> testerName=None,
>
> # Environmental variables for all steps.
> - env={},
> - extra_cmake_args=[],
> + env=None,
> + extra_cmake_args=None,
>
> # Extra repositories
> checkout_clang_tools_extra=True,
> @@ -449,9 +420,7 @@ def getClangCMakeGCSBuildFactory(
>
> # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
> stage1_upload_directory=None,
> -
> - # Triggers
> - trigger_after_stage1=[]):
> + ):
> return _getClangCMakeBuildFactory(
> clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
> vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
> @@ -462,34 +431,35 @@ def getClangCMakeGCSBuildFactory(
> checkout_clang_tools_extra=checkout_clang_tools_extra,
> checkout_compiler_rt=checkout_compiler_rt,
> stage1_upload_directory=stage1_upload_directory,
> - trigger_after_stage1=trigger_after_stage1)
> + )
> +
>
> def getClangCMakeBuildFactory(
> clean=True,
> test=True,
> - cmake='cmake',
> + cmake=None,
> jobs=None,
>
> # VS tools environment variable if using MSVC. For example,
> # %VS120COMNTOOLS% selects the 2013 toolchain.
> vs=None,
> - vs_target_arch='x86',
> + vs_target_arch=None,
>
> # Multi-stage compilation
> useTwoStage=False,
> testStage1=True,
> - stage1_config='Release',
> - stage2_config='Release',
> + stage1_config=None,
> + stage2_config=None,
>
> # Test-suite
> runTestSuite=False,
> - nt_flags=[],
> + nt_flags=None,
> submitURL=None,
> testerName=None,
>
> # Environmental variables for all steps.
> - env={},
> - extra_cmake_args=[],
> + env=None,
> + extra_cmake_args=None,
>
> # Extra repositories
> checkout_clang_tools_extra=True,
> @@ -504,32 +474,33 @@ def getClangCMakeBuildFactory(
> checkout_clang_tools_extra=checkout_clang_tools_extra,
> checkout_compiler_rt=checkout_compiler_rt)
>
> +
> def _getClangCMakeBuildFactory(
> clean=True,
> test=True,
> - cmake='cmake',
> + cmake=None,
> jobs=None,
>
> # VS tools environment variable if using MSVC. For example,
> # %VS120COMNTOOLS% selects the 2013 toolchain.
> vs=None,
> - vs_target_arch='x86',
> + vs_target_arch=None,
>
> # Multi-stage compilation
> useTwoStage=False,
> testStage1=True,
> - stage1_config='Release',
> - stage2_config='Release',
> + stage1_config=None,
> + stage2_config=None,
>
> # Test-suite
> runTestSuite=False,
> - nt_flags=[],
> + nt_flags=None,
> submitURL=None,
> testerName=None,
>
> # Environmental variables for all steps.
> - env={},
> - extra_cmake_args=[],
> + env=None,
> + extra_cmake_args=None,
>
> # Extra repositories
> checkout_clang_tools_extra=True,
> @@ -537,17 +508,44 @@ def _getClangCMakeBuildFactory(
>
> # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
> stage1_upload_directory=None,
> + ):
> +
> + if cmake is None:
> + cmake = 'cmake'
> + if env is None:
> + env = {}
> + if stage1_config is None:
> + stage1_config='Release'
>
> - # Triggers
> - trigger_after_stage1=[]):
> + if extra_cmake_args is None:
> + extra_cmake_args=[]
> +
> + depends_on_projects=['llvm', 'clang']
> + if checkout_clang_tools_extra:
> + depends_on_projects.append("clang-tools-extra")
> + if checkout_compiler_rt:
> + depends_on_projects.append("compiler-rt")
> + if runTestSuite:
> + depends_on_projects.extend(["lnt", "test-suite"])
>
> ############# PREPARING
> - f = buildbot.process.factory.BuildFactory()
> + f = LLVMBuildFactory(
> + depends_on_projects=depends_on_projects,
> + llvm_srcdir="llvm.src",
> + )
> +
> + cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
> +
> + # Do a clean checkout only if requested by the build property.
> + f.addStep(RemoveDirectory(name='clean-src-dir',
> + dir=f.llvm_srcdir,
> + haltOnFailure=False,
> + flunkOnFailure=False,
> + doStepIf=lambda step: step.build.getProperty("clean") or False,
> + ))
>
> - addSVNUpdateSteps(f,
> - checkout_clang_tools_extra=checkout_clang_tools_extra,
> - checkout_compiler_rt=checkout_compiler_rt,
> - checkout_test_suite=runTestSuite)
> + # Get the source code.
> + f.addSVNSteps()
>
> # If jobs not defined, Ninja will choose a suitable value
> jobs_cmd = []
> @@ -562,6 +560,7 @@ def _getClangCMakeBuildFactory(
> ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
> check_build_cmd = ["sh", "-c",
> "test -e build.ninja && echo OK || echo Missing"]
> +
> if vs:
> check_build_cmd = ["cmd", "/c", "if exist build.ninja (echo OK) " +
> " else (echo Missing & exit 1)"]
> @@ -574,13 +573,15 @@ def _getClangCMakeBuildFactory(
>
> # Set up VS environment, if appropriate.
> if vs:
> + if vs_target_arch is None:
> + vs_target_arch = 'x86'
> +
> f.addStep(SetProperty(
> command=builders_util.getVisualStudioEnvironment(vs, vs_target_arch),
> extract_fn=builders_util.extractSlaveEnvironment))
> assert not env, "Can't have custom builder env vars with VS"
> env = Property('slave_env')
>
> -
> ############# CLEANING
> if clean:
> f.addStep(ShellCommand(name='clean stage 1',
> @@ -649,9 +650,11 @@ def _getClangCMakeBuildFactory(
> cc = 'clang-cl.exe'
> cxx = 'clang-cl.exe'
>
> -
> ############# STAGE 2
> if useTwoStage:
> + if stage2_config is None:
> + stage2_config='Release'
> +
> # We always cleanly build the stage 2. If the compiler has been
> # changed on the stage 1, we cannot trust any of the intermediate file
> # from the old compiler. And if the stage 1 compiler is the same, we
> @@ -736,8 +739,11 @@ def _getClangCMakeBuildFactory(
> '--test-suite', test_suite_dir,
> '--cc', cc,
> '--cxx', cxx]
> +
> # Append any option provided by the user
> - test_suite_cmd.extend(nt_flags)
> + if nt_flags:
> + test_suite_cmd.extend(nt_flags)
> +
> # Only submit if a URL has been specified
> if submitURL is not None:
> if not isinstance(submitURL, list):
> @@ -784,6 +790,7 @@ def _getClangCMakeBuildFactory(
>
> return f
>
> +
> def getClangMSVCBuildFactory(update=True, clean=True, vcDrive='c', jobs=1, cmake=r"cmake"):
> f = buildbot.process.factory.BuildFactory()
>
> @@ -862,6 +869,7 @@ def getClangMSVCBuildFactory(update=True
>
> return f
>
> +
> def addClangGCCTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install",
> languages = ('gcc', 'g++', 'objc', 'obj-c++')):
> make_vars = [WithProperties(
> @@ -882,6 +890,7 @@ def addClangGCCTests(f, ignores={}, inst
> '%s.log' % lang : 'obj/%s/%s.log' % (lang, lang)},
> ignore=gcc_dg_ignores.get(lang, [])))
>
> +
> def addClangGDBTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install"):
> make_vars = [WithProperties(
> 'CC_UNDER_TEST=%s/bin/clang' % install_prefix),
> @@ -898,6 +907,7 @@ def addClangGDBTests(f, ignores={}, inst
> logfiles={ 'dg.sum' : 'obj/filtered.gdb.sum',
> 'gdb.log' : 'obj/gdb.log' }))
>
> +
> def addModernClangGDBTests(f, jobs, install_prefix):
> make_vars = [WithProperties('RUNTESTFLAGS=CC_FOR_TARGET=\'{0}/bin/clang\' '
> 'CXX_FOR_TARGET=\'{0}/bin/clang++\' '
> @@ -924,6 +934,7 @@ def addModernClangGDBTests(f, jobs, inst
> # FIXME: Deprecated.
> addClangTests = addClangGCCTests
>
> +
> def getClangTestsIgnoresFromPath(path, key):
> def readList(path):
> if not os.path.exists(path):
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list