[zorg] r280435 - First round of re-factoring to use LLVMBuildFactory for automatic scheduling.
Galina Kistanova via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 6 11:05:17 PDT 2016
I reverted it and reconfigured the master.
Thanks
Galina
On Tue, Sep 6, 2016 at 3:26 AM, Renato Golin <renato.golin at linaro.org>
wrote:
> Hi Galina,
>
> This commit (and all related refactoring changes) have completely
> broken our buildbots to a point where we can't build or work around,
> notably:
>
> * The sources are checked out in llvm.src but the build wants llvm
> * The lnt and test-suite directories are now checked in llvm.src/test,
> which is completely the wrong place
> * Check-all fails because of lit problems understanding lnt/test-suite
>
> This refactoring wasn't announced, we haven'd had time to test or
> validate, and now everything is broken. Can you please revert these
> changes and restart the master ASAP?
>
> Next steps are to make that changes locally and we should validate on
> our builders *before* it goes in production.
>
> thanks,
> --renato
>
> On 2 September 2016 at 01:16, Galina Kistanova via llvm-commits
> <llvm-commits at lists.llvm.org> wrote:
> > Author: gkistanova
> > Date: Thu Sep 1 19:16:40 2016
> > New Revision: 280435
> >
> > URL: http://llvm.org/viewvc/llvm-project?rev=280435&view=rev
> > Log:
> > First round of re-factoring to use LLVMBuildFactory for automatic
> scheduling.
> >
> > Modified:
> > zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
> >
> > Modified: zorg/trunk/zorg/buildbot/builders/ClangBuilder.py
> > URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/
> buildbot/builders/ClangBuilder.py?rev=280435&r1=280434&r2=280435&view=diff
> > ============================================================
> ==================
> > --- zorg/trunk/zorg/buildbot/builders/ClangBuilder.py (original)
> > +++ zorg/trunk/zorg/buildbot/builders/ClangBuilder.py Thu Sep 1
> 19:16:40 2016
> > @@ -8,8 +8,11 @@ from buildbot.process.properties import
> > from buildbot.steps.shell import Configure, ShellCommand, SetProperty
> > from buildbot.steps.shell import WarningCountingShellCommand
> > from buildbot.steps.source import SVN
> > +from buildbot.steps.slave import RemoveDirectory
> > from buildbot.steps.transfer import FileDownload
> >
> > +from zorg.buildbot.process.factory import LLVMBuildFactory
> > +
> > import zorg.buildbot.util.artifacts as artifacts
> > import zorg.buildbot.builders.Util as builders_util
> > import zorg.buildbot.util.phasedbuilderutils as phasedbuilderutils
> > @@ -17,6 +20,7 @@ import zorg.buildbot.commands as command
> > import zorg.buildbot.commands.BatchFileDownload as batch_file_download
> > import zorg.buildbot.commands.LitTestCommand as lit_test_command
> >
> > +
> > def getClangBuildFactory(
> > triple=None,
> > clean=True,
> > @@ -335,40 +339,6 @@ def getClangBuildFactory(
> >
> > return f
> >
> > -def addSVNUpdateSteps(f,
> > - checkout_clang_tools_extra,
> > - checkout_compiler_rt,
> > - checkout_test_suite):
> > - # We *must* checkout at least Clang+LLVM
> > - f.addStep(SVN(name='svn-llvm',
> > - mode='update', baseURL='http://llvm.org/svn/
> llvm-project/llvm/',
> > - defaultBranch='trunk',
> > - workdir='llvm'))
> > - f.addStep(SVN(name='svn-clang',
> > - mode='update', baseURL='http://llvm.org/svn/
> llvm-project/cfe/',
> > - defaultBranch='trunk',
> > - workdir='llvm/tools/clang'))
> > -
> > - # Extra stuff that will be built/tested
> > - if checkout_clang_tools_extra:
> > - f.addStep(SVN(name='svn-clang-tools-extra',
> > - mode='update', baseURL='http://llvm.org/svn/
> llvm-project/clang-tools-extra/',
> > - defaultBranch='trunk',
> > - workdir='llvm/tools/clang/tools/extra'))
> > - if checkout_compiler_rt:
> > - f.addStep(SVN(name='svn-compiler-rt',
> > - mode='update', baseURL='http://llvm.org/svn/
> llvm-project/compiler-rt/',
> > - defaultBranch='trunk',
> > - workdir='llvm/projects/compiler-rt'))
> > - if checkout_test_suite:
> > - f.addStep(SVN(name='svn-lnt',
> > - mode='update', baseURL='http://llvm.org/svn/
> llvm-project/lnt/',
> > - defaultBranch='trunk',
> > - workdir='test/lnt'))
> > - f.addStep(SVN(name='svn-test-suite',
> > - mode='update', baseURL='http://llvm.org/svn/
> llvm-project/test-suite/',
> > - defaultBranch='trunk',
> > - workdir='test/test-suite'))
> >
> > def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory,
> env,
> > gcs_url_property=None):
> > @@ -416,32 +386,33 @@ def addGCSUploadSteps(f, package_name, i
> > workdir=install_prefix,
> > env=env))
> >
> > +
> > def getClangCMakeGCSBuildFactory(
> > clean=True,
> > test=True,
> > - cmake='cmake',
> > + cmake=None,
> > jobs=None,
> >
> > # VS tools environment variable if using MSVC. For example,
> > # %VS120COMNTOOLS% selects the 2013 toolchain.
> > vs=None,
> > - vs_target_arch='x86',
> > + vs_target_arch=None,
> >
> > # Multi-stage compilation
> > useTwoStage=False,
> > testStage1=True,
> > - stage1_config='Release',
> > - stage2_config='Release',
> > + stage1_config=None,
> > + stage2_config=None,
> >
> > # Test-suite
> > runTestSuite=False,
> > - nt_flags=[],
> > + nt_flags=None,
> > submitURL=None,
> > testerName=None,
> >
> > # Environmental variables for all steps.
> > - env={},
> > - extra_cmake_args=[],
> > + env=None,
> > + extra_cmake_args=None,
> >
> > # Extra repositories
> > checkout_clang_tools_extra=True,
> > @@ -449,9 +420,7 @@ def getClangCMakeGCSBuildFactory(
> >
> > # Upload artifacts to Google Cloud Storage (for the
> llvmbisect tool)
> > stage1_upload_directory=None,
> > -
> > - # Triggers
> > - trigger_after_stage1=[]):
> > + ):
> > return _getClangCMakeBuildFactory(
> > clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
> > vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
> > @@ -462,34 +431,35 @@ def getClangCMakeGCSBuildFactory(
> > checkout_clang_tools_extra=checkout_clang_tools_extra,
> > checkout_compiler_rt=checkout_compiler_rt,
> > stage1_upload_directory=stage1_upload_directory,
> > - trigger_after_stage1=trigger_after_stage1)
> > + )
> > +
> >
> > def getClangCMakeBuildFactory(
> > clean=True,
> > test=True,
> > - cmake='cmake',
> > + cmake=None,
> > jobs=None,
> >
> > # VS tools environment variable if using MSVC. For example,
> > # %VS120COMNTOOLS% selects the 2013 toolchain.
> > vs=None,
> > - vs_target_arch='x86',
> > + vs_target_arch=None,
> >
> > # Multi-stage compilation
> > useTwoStage=False,
> > testStage1=True,
> > - stage1_config='Release',
> > - stage2_config='Release',
> > + stage1_config=None,
> > + stage2_config=None,
> >
> > # Test-suite
> > runTestSuite=False,
> > - nt_flags=[],
> > + nt_flags=None,
> > submitURL=None,
> > testerName=None,
> >
> > # Environmental variables for all steps.
> > - env={},
> > - extra_cmake_args=[],
> > + env=None,
> > + extra_cmake_args=None,
> >
> > # Extra repositories
> > checkout_clang_tools_extra=True,
> > @@ -504,32 +474,33 @@ def getClangCMakeBuildFactory(
> > checkout_clang_tools_extra=checkout_clang_tools_extra,
> > checkout_compiler_rt=checkout_compiler_rt)
> >
> > +
> > def _getClangCMakeBuildFactory(
> > clean=True,
> > test=True,
> > - cmake='cmake',
> > + cmake=None,
> > jobs=None,
> >
> > # VS tools environment variable if using MSVC. For example,
> > # %VS120COMNTOOLS% selects the 2013 toolchain.
> > vs=None,
> > - vs_target_arch='x86',
> > + vs_target_arch=None,
> >
> > # Multi-stage compilation
> > useTwoStage=False,
> > testStage1=True,
> > - stage1_config='Release',
> > - stage2_config='Release',
> > + stage1_config=None,
> > + stage2_config=None,
> >
> > # Test-suite
> > runTestSuite=False,
> > - nt_flags=[],
> > + nt_flags=None,
> > submitURL=None,
> > testerName=None,
> >
> > # Environmental variables for all steps.
> > - env={},
> > - extra_cmake_args=[],
> > + env=None,
> > + extra_cmake_args=None,
> >
> > # Extra repositories
> > checkout_clang_tools_extra=True,
> > @@ -537,17 +508,44 @@ def _getClangCMakeBuildFactory(
> >
> > # Upload artifacts to Google Cloud Storage (for the
> llvmbisect tool)
> > stage1_upload_directory=None,
> > + ):
> > +
> > + if cmake is None:
> > + cmake = 'cmake'
> > + if env is None:
> > + env = {}
> > + if stage1_config is None:
> > + stage1_config='Release'
> >
> > - # Triggers
> > - trigger_after_stage1=[]):
> > + if extra_cmake_args is None:
> > + extra_cmake_args=[]
> > +
> > + depends_on_projects=['llvm', 'clang']
> > + if checkout_clang_tools_extra:
> > + depends_on_projects.append("clang-tools-extra")
> > + if checkout_compiler_rt:
> > + depends_on_projects.append("compiler-rt")
> > + if runTestSuite:
> > + depends_on_projects.extend(["lnt", "test-suite"])
> >
> > ############# PREPARING
> > - f = buildbot.process.factory.BuildFactory()
> > + f = LLVMBuildFactory(
> > + depends_on_projects=depends_on_projects,
> > + llvm_srcdir="llvm.src",
> > + )
> > +
> > + cleanBuildRequested = lambda step: step.build.getProperty("clean")
> or clean
> > +
> > + # Do a clean checkout only if requested by the build property.
> > + f.addStep(RemoveDirectory(name='clean-src-dir',
> > + dir=f.llvm_srcdir,
> > + haltOnFailure=False,
> > + flunkOnFailure=False,
> > + doStepIf=lambda step: step.build.getProperty("clean") or
> False,
> > + ))
> >
> > - addSVNUpdateSteps(f,
> > - checkout_clang_tools_extra=
> checkout_clang_tools_extra,
> > - checkout_compiler_rt=checkout_compiler_rt,
> > - checkout_test_suite=runTestSuite)
> > + # Get the source code.
> > + f.addSVNSteps()
> >
> > # If jobs not defined, Ninja will choose a suitable value
> > jobs_cmd = []
> > @@ -562,6 +560,7 @@ def _getClangCMakeBuildFactory(
> > ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
> > check_build_cmd = ["sh", "-c",
> > "test -e build.ninja && echo OK || echo Missing"]
> > +
> > if vs:
> > check_build_cmd = ["cmd", "/c", "if exist build.ninja (echo OK)
> " +
> > " else (echo Missing & exit 1)"]
> > @@ -574,13 +573,15 @@ def _getClangCMakeBuildFactory(
> >
> > # Set up VS environment, if appropriate.
> > if vs:
> > + if vs_target_arch is None:
> > + vs_target_arch = 'x86'
> > +
> > f.addStep(SetProperty(
> > command=builders_util.getVisualStudioEnvironment(vs,
> vs_target_arch),
> > extract_fn=builders_util.extractSlaveEnvironment))
> > assert not env, "Can't have custom builder env vars with VS"
> > env = Property('slave_env')
> >
> > -
> > ############# CLEANING
> > if clean:
> > f.addStep(ShellCommand(name='clean stage 1',
> > @@ -649,9 +650,11 @@ def _getClangCMakeBuildFactory(
> > cc = 'clang-cl.exe'
> > cxx = 'clang-cl.exe'
> >
> > -
> > ############# STAGE 2
> > if useTwoStage:
> > + if stage2_config is None:
> > + stage2_config='Release'
> > +
> > # We always cleanly build the stage 2. If the compiler has been
> > # changed on the stage 1, we cannot trust any of the
> intermediate file
> > # from the old compiler. And if the stage 1 compiler is the
> same, we
> > @@ -736,8 +739,11 @@ def _getClangCMakeBuildFactory(
> > '--test-suite', test_suite_dir,
> > '--cc', cc,
> > '--cxx', cxx]
> > +
> > # Append any option provided by the user
> > - test_suite_cmd.extend(nt_flags)
> > + if nt_flags:
> > + test_suite_cmd.extend(nt_flags)
> > +
> > # Only submit if a URL has been specified
> > if submitURL is not None:
> > if not isinstance(submitURL, list):
> > @@ -784,6 +790,7 @@ def _getClangCMakeBuildFactory(
> >
> > return f
> >
> > +
> > def getClangMSVCBuildFactory(update=True, clean=True, vcDrive='c',
> jobs=1, cmake=r"cmake"):
> > f = buildbot.process.factory.BuildFactory()
> >
> > @@ -862,6 +869,7 @@ def getClangMSVCBuildFactory(update=True
> >
> > return f
> >
> > +
> > def addClangGCCTests(f, ignores={}, install_prefix="%(builddir)s/
> llvm.install",
> > languages = ('gcc', 'g++', 'objc', 'obj-c++')):
> > make_vars = [WithProperties(
> > @@ -882,6 +890,7 @@ def addClangGCCTests(f, ignores={}, inst
> > '%s.log' % lang : 'obj/%s/%s.log' % (lang,
> lang)},
> > ignore=gcc_dg_ignores.get(lang, [])))
> >
> > +
> > def addClangGDBTests(f, ignores={}, install_prefix="%(builddir)s/
> llvm.install"):
> > make_vars = [WithProperties(
> > 'CC_UNDER_TEST=%s/bin/clang' % install_prefix),
> > @@ -898,6 +907,7 @@ def addClangGDBTests(f, ignores={}, inst
> > logfiles={ 'dg.sum' : 'obj/filtered.gdb.sum',
> > 'gdb.log' : 'obj/gdb.log' }))
> >
> > +
> > def addModernClangGDBTests(f, jobs, install_prefix):
> > make_vars = [WithProperties('RUNTESTFLAGS=
> CC_FOR_TARGET=\'{0}/bin/clang\' '
> > 'CXX_FOR_TARGET=\'{0}/bin/clang++\' '
> > @@ -924,6 +934,7 @@ def addModernClangGDBTests(f, jobs, inst
> > # FIXME: Deprecated.
> > addClangTests = addClangGCCTests
> >
> > +
> > def getClangTestsIgnoresFromPath(path, key):
> > def readList(path):
> > if not os.path.exists(path):
> >
> >
> > _______________________________________________
> > llvm-commits mailing list
> > llvm-commits at lists.llvm.org
> > http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20160906/ba781ff6/attachment.html>
More information about the llvm-commits
mailing list