<div dir="ltr"><div><div>I reverted it and reconfigured the master.<br><br></div>Thanks<br><br></div>Galina<br></div><div class="gmail_extra"><br><div class="gmail_quote">On Tue, Sep 6, 2016 at 3:26 AM, Renato Golin <span dir="ltr"><<a href="mailto:renato.golin@linaro.org" target="_blank">renato.golin@linaro.org</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Hi Galina,<br>
<br>
This commit (and all related refactoring changes) have completely<br>
broken our buildbots to a point where we can't build or work around,<br>
notably:<br>
<br>
* The sources are checked out in llvm.src but the build wants llvm<br>
* The lnt and test-suite directories are now checked in llvm.src/test,<br>
which is completely the wrong place<br>
* Check-all fails because of lit problems understanding lnt/test-suite<br>
<br>
This refactoring wasn't announced, we haven'd had time to test or<br>
validate, and now everything is broken. Can you please revert these<br>
changes and restart the master ASAP?<br>
<br>
Next steps are to make that changes locally and we should validate on<br>
our builders *before* it goes in production.<br>
<br>
thanks,<br>
--renato<br>
<br>
On 2 September 2016 at 01:16, Galina Kistanova via llvm-commits<br>
<div class="HOEnZb"><div class="h5"><<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a>> wrote:<br>
> Author: gkistanova<br>
> Date: Thu Sep 1 19:16:40 2016<br>
> New Revision: 280435<br>
><br>
> URL: <a href="http://llvm.org/viewvc/llvm-project?rev=280435&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project?rev=280435&view=rev</a><br>
> Log:<br>
> First round of re-factoring to use LLVMBuildFactory for automatic scheduling.<br>
><br>
> Modified:<br>
> zorg/trunk/zorg/buildbot/<wbr>builders/ClangBuilder.py<br>
><br>
> Modified: zorg/trunk/zorg/buildbot/<wbr>builders/ClangBuilder.py<br>
> URL: <a href="http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/builders/ClangBuilder.py?rev=280435&r1=280434&r2=280435&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project/zorg/trunk/zorg/<wbr>buildbot/builders/<wbr>ClangBuilder.py?rev=280435&r1=<wbr>280434&r2=280435&view=diff</a><br>
> ==============================<wbr>==============================<wbr>==================<br>
> --- zorg/trunk/zorg/buildbot/<wbr>builders/ClangBuilder.py (original)<br>
> +++ zorg/trunk/zorg/buildbot/<wbr>builders/ClangBuilder.py Thu Sep 1 19:16:40 2016<br>
> @@ -8,8 +8,11 @@ from buildbot.process.properties import<br>
> from buildbot.steps.shell import Configure, ShellCommand, SetProperty<br>
> from buildbot.steps.shell import WarningCountingShellCommand<br>
> from buildbot.steps.source import SVN<br>
> +from buildbot.steps.slave import RemoveDirectory<br>
> from buildbot.steps.transfer import FileDownload<br>
><br>
> +from zorg.buildbot.process.factory import LLVMBuildFactory<br>
> +<br>
> import zorg.buildbot.util.artifacts as artifacts<br>
> import zorg.buildbot.builders.Util as builders_util<br>
> import zorg.buildbot.util.<wbr>phasedbuilderutils as phasedbuilderutils<br>
> @@ -17,6 +20,7 @@ import zorg.buildbot.commands as command<br>
> import zorg.buildbot.commands.<wbr>BatchFileDownload as batch_file_download<br>
> import zorg.buildbot.commands.<wbr>LitTestCommand as lit_test_command<br>
><br>
> +<br>
> def getClangBuildFactory(<br>
> triple=None,<br>
> clean=True,<br>
> @@ -335,40 +339,6 @@ def getClangBuildFactory(<br>
><br>
> return f<br>
><br>
> -def addSVNUpdateSteps(f,<br>
> - checkout_clang_tools_extra,<br>
> - checkout_compiler_rt,<br>
> - checkout_test_suite):<br>
> - # We *must* checkout at least Clang+LLVM<br>
> - f.addStep(SVN(name='svn-llvm',<br>
> - mode='update', baseURL='<a href="http://llvm.org/svn/llvm-project/llvm/" rel="noreferrer" target="_blank">http://llvm.org/svn/<wbr>llvm-project/llvm/</a>',<br>
> - defaultBranch='trunk',<br>
> - workdir='llvm'))<br>
> - f.addStep(SVN(name='svn-clang'<wbr>,<br>
> - mode='update', baseURL='<a href="http://llvm.org/svn/llvm-project/cfe/" rel="noreferrer" target="_blank">http://llvm.org/svn/<wbr>llvm-project/cfe/</a>',<br>
> - defaultBranch='trunk',<br>
> - workdir='llvm/tools/clang'))<br>
> -<br>
> - # Extra stuff that will be built/tested<br>
> - if checkout_clang_tools_extra:<br>
> - f.addStep(SVN(name='svn-clang-<wbr>tools-extra',<br>
> - mode='update', baseURL='<a href="http://llvm.org/svn/llvm-project/clang-tools-extra/" rel="noreferrer" target="_blank">http://llvm.org/svn/<wbr>llvm-project/clang-tools-<wbr>extra/</a>',<br>
> - defaultBranch='trunk',<br>
> - workdir='llvm/tools/clang/<wbr>tools/extra'))<br>
> - if checkout_compiler_rt:<br>
> - f.addStep(SVN(name='svn-<wbr>compiler-rt',<br>
> - mode='update', baseURL='<a href="http://llvm.org/svn/llvm-project/compiler-rt/" rel="noreferrer" target="_blank">http://llvm.org/svn/<wbr>llvm-project/compiler-rt/</a>',<br>
> - defaultBranch='trunk',<br>
> - workdir='llvm/projects/<wbr>compiler-rt'))<br>
> - if checkout_test_suite:<br>
> - f.addStep(SVN(name='svn-lnt',<br>
> - mode='update', baseURL='<a href="http://llvm.org/svn/llvm-project/lnt/" rel="noreferrer" target="_blank">http://llvm.org/svn/<wbr>llvm-project/lnt/</a>',<br>
> - defaultBranch='trunk',<br>
> - workdir='test/lnt'))<br>
> - f.addStep(SVN(name='svn-test-<wbr>suite',<br>
> - mode='update', baseURL='<a href="http://llvm.org/svn/llvm-project/test-suite/" rel="noreferrer" target="_blank">http://llvm.org/svn/<wbr>llvm-project/test-suite/</a>',<br>
> - defaultBranch='trunk',<br>
> - workdir='test/test-suite'))<br>
><br>
> def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env,<br>
> gcs_url_property=None):<br>
> @@ -416,32 +386,33 @@ def addGCSUploadSteps(f, package_name, i<br>
> workdir=install_prefix,<br>
> env=env))<br>
><br>
> +<br>
> def getClangCMakeGCSBuildFactory(<br>
> clean=True,<br>
> test=True,<br>
> - cmake='cmake',<br>
> + cmake=None,<br>
> jobs=None,<br>
><br>
> # VS tools environment variable if using MSVC. For example,<br>
> # %VS120COMNTOOLS% selects the 2013 toolchain.<br>
> vs=None,<br>
> - vs_target_arch='x86',<br>
> + vs_target_arch=None,<br>
><br>
> # Multi-stage compilation<br>
> useTwoStage=False,<br>
> testStage1=True,<br>
> - stage1_config='Release',<br>
> - stage2_config='Release',<br>
> + stage1_config=None,<br>
> + stage2_config=None,<br>
><br>
> # Test-suite<br>
> runTestSuite=False,<br>
> - nt_flags=[],<br>
> + nt_flags=None,<br>
> submitURL=None,<br>
> testerName=None,<br>
><br>
> # Environmental variables for all steps.<br>
> - env={},<br>
> - extra_cmake_args=[],<br>
> + env=None,<br>
> + extra_cmake_args=None,<br>
><br>
> # Extra repositories<br>
> checkout_clang_tools_extra=<wbr>True,<br>
> @@ -449,9 +420,7 @@ def getClangCMakeGCSBuildFactory(<br>
><br>
> # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)<br>
> stage1_upload_directory=None,<br>
> -<br>
> - # Triggers<br>
> - trigger_after_stage1=[]):<br>
> + ):<br>
> return _getClangCMakeBuildFactory(<br>
> clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,<br>
> vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,<br>
> @@ -462,34 +431,35 @@ def getClangCMakeGCSBuildFactory(<br>
> checkout_clang_tools_extra=<wbr>checkout_clang_tools_extra,<br>
> checkout_compiler_rt=checkout_<wbr>compiler_rt,<br>
> stage1_upload_directory=<wbr>stage1_upload_directory,<br>
> - trigger_after_stage1=trigger_<wbr>after_stage1)<br>
> + )<br>
> +<br>
><br>
> def getClangCMakeBuildFactory(<br>
> clean=True,<br>
> test=True,<br>
> - cmake='cmake',<br>
> + cmake=None,<br>
> jobs=None,<br>
><br>
> # VS tools environment variable if using MSVC. For example,<br>
> # %VS120COMNTOOLS% selects the 2013 toolchain.<br>
> vs=None,<br>
> - vs_target_arch='x86',<br>
> + vs_target_arch=None,<br>
><br>
> # Multi-stage compilation<br>
> useTwoStage=False,<br>
> testStage1=True,<br>
> - stage1_config='Release',<br>
> - stage2_config='Release',<br>
> + stage1_config=None,<br>
> + stage2_config=None,<br>
><br>
> # Test-suite<br>
> runTestSuite=False,<br>
> - nt_flags=[],<br>
> + nt_flags=None,<br>
> submitURL=None,<br>
> testerName=None,<br>
><br>
> # Environmental variables for all steps.<br>
> - env={},<br>
> - extra_cmake_args=[],<br>
> + env=None,<br>
> + extra_cmake_args=None,<br>
><br>
> # Extra repositories<br>
> checkout_clang_tools_extra=<wbr>True,<br>
> @@ -504,32 +474,33 @@ def getClangCMakeBuildFactory(<br>
> checkout_clang_tools_extra=<wbr>checkout_clang_tools_extra,<br>
> checkout_compiler_rt=checkout_<wbr>compiler_rt)<br>
><br>
> +<br>
> def _getClangCMakeBuildFactory(<br>
> clean=True,<br>
> test=True,<br>
> - cmake='cmake',<br>
> + cmake=None,<br>
> jobs=None,<br>
><br>
> # VS tools environment variable if using MSVC. For example,<br>
> # %VS120COMNTOOLS% selects the 2013 toolchain.<br>
> vs=None,<br>
> - vs_target_arch='x86',<br>
> + vs_target_arch=None,<br>
><br>
> # Multi-stage compilation<br>
> useTwoStage=False,<br>
> testStage1=True,<br>
> - stage1_config='Release',<br>
> - stage2_config='Release',<br>
> + stage1_config=None,<br>
> + stage2_config=None,<br>
><br>
> # Test-suite<br>
> runTestSuite=False,<br>
> - nt_flags=[],<br>
> + nt_flags=None,<br>
> submitURL=None,<br>
> testerName=None,<br>
><br>
> # Environmental variables for all steps.<br>
> - env={},<br>
> - extra_cmake_args=[],<br>
> + env=None,<br>
> + extra_cmake_args=None,<br>
><br>
> # Extra repositories<br>
> checkout_clang_tools_extra=<wbr>True,<br>
> @@ -537,17 +508,44 @@ def _getClangCMakeBuildFactory(<br>
><br>
> # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)<br>
> stage1_upload_directory=None,<br>
> + ):<br>
> +<br>
> + if cmake is None:<br>
> + cmake = 'cmake'<br>
> + if env is None:<br>
> + env = {}<br>
> + if stage1_config is None:<br>
> + stage1_config='Release'<br>
><br>
> - # Triggers<br>
> - trigger_after_stage1=[]):<br>
> + if extra_cmake_args is None:<br>
> + extra_cmake_args=[]<br>
> +<br>
> + depends_on_projects=['llvm', 'clang']<br>
> + if checkout_clang_tools_extra:<br>
> + depends_on_projects.append("<wbr>clang-tools-extra")<br>
> + if checkout_compiler_rt:<br>
> + depends_on_projects.append("<wbr>compiler-rt")<br>
> + if runTestSuite:<br>
> + depends_on_projects.extend(["<wbr>lnt", "test-suite"])<br>
><br>
> ############# PREPARING<br>
> - f = buildbot.process.factory.<wbr>BuildFactory()<br>
> + f = LLVMBuildFactory(<br>
> + depends_on_projects=depends_<wbr>on_projects,<br>
> + llvm_srcdir="llvm.src",<br>
> + )<br>
> +<br>
> + cleanBuildRequested = lambda step: step.build.getProperty("clean"<wbr>) or clean<br>
> +<br>
> + # Do a clean checkout only if requested by the build property.<br>
> + f.addStep(RemoveDirectory(<wbr>name='clean-src-dir',<br>
> + dir=f.llvm_srcdir,<br>
> + haltOnFailure=False,<br>
> + flunkOnFailure=False,<br>
> + doStepIf=lambda step: step.build.getProperty("clean"<wbr>) or False,<br>
> + ))<br>
><br>
> - addSVNUpdateSteps(f,<br>
> - checkout_clang_tools_extra=<wbr>checkout_clang_tools_extra,<br>
> - checkout_compiler_rt=checkout_<wbr>compiler_rt,<br>
> - checkout_test_suite=<wbr>runTestSuite)<br>
> + # Get the source code.<br>
> + f.addSVNSteps()<br>
><br>
> # If jobs not defined, Ninja will choose a suitable value<br>
> jobs_cmd = []<br>
> @@ -562,6 +560,7 @@ def _getClangCMakeBuildFactory(<br>
> ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd<br>
> check_build_cmd = ["sh", "-c",<br>
> "test -e build.ninja && echo OK || echo Missing"]<br>
> +<br>
> if vs:<br>
> check_build_cmd = ["cmd", "/c", "if exist build.ninja (echo OK) " +<br>
> " else (echo Missing & exit 1)"]<br>
> @@ -574,13 +573,15 @@ def _getClangCMakeBuildFactory(<br>
><br>
> # Set up VS environment, if appropriate.<br>
> if vs:<br>
> + if vs_target_arch is None:<br>
> + vs_target_arch = 'x86'<br>
> +<br>
> f.addStep(SetProperty(<br>
> command=builders_util.<wbr>getVisualStudioEnvironment(vs, vs_target_arch),<br>
> extract_fn=builders_util.<wbr>extractSlaveEnvironment))<br>
> assert not env, "Can't have custom builder env vars with VS"<br>
> env = Property('slave_env')<br>
><br>
> -<br>
> ############# CLEANING<br>
> if clean:<br>
> f.addStep(ShellCommand(name='<wbr>clean stage 1',<br>
> @@ -649,9 +650,11 @@ def _getClangCMakeBuildFactory(<br>
> cc = 'clang-cl.exe'<br>
> cxx = 'clang-cl.exe'<br>
><br>
> -<br>
> ############# STAGE 2<br>
> if useTwoStage:<br>
> + if stage2_config is None:<br>
> + stage2_config='Release'<br>
> +<br>
> # We always cleanly build the stage 2. If the compiler has been<br>
> # changed on the stage 1, we cannot trust any of the intermediate file<br>
> # from the old compiler. And if the stage 1 compiler is the same, we<br>
> @@ -736,8 +739,11 @@ def _getClangCMakeBuildFactory(<br>
> '--test-suite', test_suite_dir,<br>
> '--cc', cc,<br>
> '--cxx', cxx]<br>
> +<br>
> # Append any option provided by the user<br>
> - test_suite_cmd.extend(nt_<wbr>flags)<br>
> + if nt_flags:<br>
> + test_suite_cmd.extend(nt_<wbr>flags)<br>
> +<br>
> # Only submit if a URL has been specified<br>
> if submitURL is not None:<br>
> if not isinstance(submitURL, list):<br>
> @@ -784,6 +790,7 @@ def _getClangCMakeBuildFactory(<br>
><br>
> return f<br>
><br>
> +<br>
> def getClangMSVCBuildFactory(<wbr>update=True, clean=True, vcDrive='c', jobs=1, cmake=r"cmake"):<br>
> f = buildbot.process.factory.<wbr>BuildFactory()<br>
><br>
> @@ -862,6 +869,7 @@ def getClangMSVCBuildFactory(<wbr>update=True<br>
><br>
> return f<br>
><br>
> +<br>
> def addClangGCCTests(f, ignores={}, install_prefix="%(builddir)s/<wbr>llvm.install",<br>
> languages = ('gcc', 'g++', 'objc', 'obj-c++')):<br>
> make_vars = [WithProperties(<br>
> @@ -882,6 +890,7 @@ def addClangGCCTests(f, ignores={}, inst<br>
> '%s.log' % lang : 'obj/%s/%s.log' % (lang, lang)},<br>
> ignore=gcc_dg_ignores.get(<wbr>lang, [])))<br>
><br>
> +<br>
> def addClangGDBTests(f, ignores={}, install_prefix="%(builddir)s/<wbr>llvm.install"):<br>
> make_vars = [WithProperties(<br>
> 'CC_UNDER_TEST=%s/bin/clang' % install_prefix),<br>
> @@ -898,6 +907,7 @@ def addClangGDBTests(f, ignores={}, inst<br>
> logfiles={ 'dg.sum' : 'obj/filtered.gdb.sum',<br>
> 'gdb.log' : 'obj/gdb.log' }))<br>
><br>
> +<br>
> def addModernClangGDBTests(f, jobs, install_prefix):<br>
> make_vars = [WithProperties('RUNTESTFLAGS=<wbr>CC_FOR_TARGET=\'{0}/bin/clang\<wbr>' '<br>
> 'CXX_FOR_TARGET=\'{0}/bin/<wbr>clang++\' '<br>
> @@ -924,6 +934,7 @@ def addModernClangGDBTests(f, jobs, inst<br>
> # FIXME: Deprecated.<br>
> addClangTests = addClangGCCTests<br>
><br>
> +<br>
> def getClangTestsIgnoresFromPath(<wbr>path, key):<br>
> def readList(path):<br>
> if not os.path.exists(path):<br>
><br>
><br>
> ______________________________<wbr>_________________<br>
> llvm-commits mailing list<br>
> <a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a><br>
> <a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/<wbr>mailman/listinfo/llvm-commits</a><br>
</div></div></blockquote></div><br></div>