[zorg] r366297 - Revert "[zorg] Add a new build script for use with monorepo"
Azharuddin Mohammed via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 17 00:49:12 PDT 2019
Author: azharudd
Date: Wed Jul 17 00:49:12 2019
New Revision: 366297
URL: http://llvm.org/viewvc/llvm-project?rev=366297&view=rev
Log:
Revert "[zorg] Add a new build script for use with monorepo"
This reverts commit c842a5a3fa0d0d7d00e206c47d58fba7288111d1 (r365440).
Removed:
zorg/trunk/test/jenkins/test_monorepo_build.py
zorg/trunk/zorg/jenkins/monorepo_build.py
Removed: zorg/trunk/test/jenkins/test_monorepo_build.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/test/jenkins/test_monorepo_build.py?rev=366296&view=auto
==============================================================================
--- zorg/trunk/test/jenkins/test_monorepo_build.py (original)
+++ zorg/trunk/test/jenkins/test_monorepo_build.py (removed)
@@ -1,132 +0,0 @@
-# Testing for the Jenkins monorepo_build.py script
-#
-# RUN: export TESTING=1
-# RUN: export JOB_NAME="FOO"
-# RUN: export BUILD_NUMBER=321
-# RUN: export BRANCH=master
-# Tell monorepo_build.py to just print commands instead of running.
-# RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/libcxx %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb
-# RUN: touch %t.SANDBOX/host-compiler/bin/clang
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all > %t.log
-# RUN: FileCheck --check-prefix CHECK-SIMPLE < %t.log %s
-# CHECK-SIMPLE: @@@ Build Directory @@@
-# CHECK-SIMPLE: cd
-# CHECK-SIMPLE: 'mkdir' '-p'
-# CHECK-SIMPLE: @@@@@@
-# CHECK-SIMPLE: @@@ Build Clang @@@
-# CHECK-SIMPLE: cd
-# CHECK-SIMPLE: 'mkdir' './Build' './Root'
-# CHECK-SIMPLE: cd
-# CHECK-SIMPLE: '/usr/local/bin/cmake' '-G' 'Ninja' '-C'
-# CHECK-SIMPLE: '-DLLVM_ENABLE_ASSERTIONS:BOOL=FALSE'
-# CHECK-SIMPLE: '-DCMAKE_BUILD_TYPE=RelWithDebInfo'
-# CHECK-SIMPLE: '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;compiler-rt;libcxx'
-# CHECK-SIMPLE: '-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja'
-# CHECK-SIMPLE: '-DLLVM_VERSION_PATCH=99'
-# CHECK-SIMPLE: '-DLLVM_VERSION_SUFFIX=""'
-# CHECK-SIMPLE: '-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On'
-# CHECK-SIMPLE: '-DCLANG_COMPILER_RT_CMAKE_ARGS
-# CHECK-SIMPLE: Apple.cmake'
-# CHECK-SIMPLE: '-DCOMPILER_RT_BUILD_SANITIZERS=On'
-# CHECK-SIMPLE: '-DCMAKE_INSTALL_PREFIX
-# CHECK-SIMPLE: '-DCLANG_APPEND_VC_REV=On'
-# CHECK-SIMPLE: '-DLLVM_BUILD_TESTS=On'
-# CHECK-SIMPLE: '-DLLVM_INCLUDE_TESTS=On'
-# CHECK-SIMPLE: '-DCLANG_INCLUDE_TESTS=On'
-# CHECK-SIMPLE: '-DLLVM_INCLUDE_UTILS=On'
-# CHECK-SIMPLE: '-DCMAKE_MACOSX_RPATH=On'
-# CHECK-SIMPLE: '-DLLVM_ENABLE_LTO=Off
-# CHECK-SIMPLE-NOT: -DLLVM_PARALLEL_LINK_JOBS
-# CHECK-SIMPLE: @@@@@@
-# CHECK-SIMPLE: @@@ Ninja @@@
-# CHECK-SIMPLE: cd
-# CHECK-SIMPLE: '/usr/local/bin/ninja' '-v' 'install'
-# CHECK-SIMPLE: @@@@@@
-# CHECK-SIMPLE: @@@ Tests @@@
-
-# CHECK-SIMPLE: cd
-# CHECK-SIMPLE: 'env' 'MALLOC_LOG_FILE=/dev/null' '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all'
-
-# Now Check Assertion Builds have --enable assertions
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --assertions > %t-assert.log
-# RUN: FileCheck --check-prefix CHECK-ASSERT < %t-assert.log %s
-# CHECK-ASSERT: '/usr/local/bin/cmake' '-G' 'Ninja' '-C'
-# CHECK-ASSERT: '-DLLVM_ENABLE_ASSERTIONS:BOOL=TRUE'
-
-# Check LTO
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --lto > %t-lto.log
-# RUN: FileCheck --check-prefix CHECK-LTO < %t-lto.log %s
-# CHECK-LTO: -DLLVM_PARALLEL_LINK_JOBS
-# CHECK-LTO-NOT:: '-DLLVM_ENABLE_LTO=Off
-
-# Now try just a build
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang build --lto
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang build
-
-# Just a test
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang test
-
-# CMake
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --debug > %t-cmake.log
-# RUN: FileCheck --check-prefix CHECK-CMAKE < %t-cmake.log %s
-# CHECK-CMAKE: '/usr/local/bin/cmake' '-G' 'Ninja'
-# CHECK-CMAKE: -DLLVM_BUILD_EXAMPLES=On
-# CHECK-CMAKE: '-DCMAKE_BUILD_TYPE=Debug'
-# CHECK-CMAKE: '-DLLVM_ENABLE_ASSERTIONS=Off'
-# CHECK-CMAKE: -DLLVM_LIT_ARGS=--xunit-xml-output=testresults.xunit.xml -v --timeout=600
-# CHECK-CMAKE: '/usr/local/bin/ninja' '-v' 'all'
-# CHECK-CMAKE: '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all'
-
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake build
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake test
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake testlong
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --lto | FileCheck --check-prefix CHECK-CMAKELTO %s
-# CHECK-CMAKELTO: '/usr/local/bin/cmake' '-G' 'Ninja'
-# CHECK-CMAKELTO: '-DLLVM_BUILD_EXAMPLES=Off'
-# CHECK-CMAKELTO-NOT:: '-DLLVM_ENABLE_LTO=Off
-# CHECK-CMAKELTO: '-DLLVM_PARALLEL_LINK_JOBS=1'
-# CHECK-CMAKELTO: '-DCMAKE_BUILD_TYPE=Release'
-
-# RUN: env MAX_PARALLEL_LINKS=2 python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --lto | FileCheck --check-prefix CHECK-CMAKE-PAR-LTO %s
-# CHECK-CMAKE-PAR-LTO: '/usr/local/bin/cmake' '-G' 'Ninja'
-# CHECK-CMAKE-PAR-LTO: '-DLLVM_BUILD_EXAMPLES=Off'
-# CHECK-CMAKE-PAR-LTO-NOT:: '-DLLVM_ENABLE_LTO=Off
-# CHECK-CMAKE-PAR-LTO: '-DLLVM_PARALLEL_LINK_JOBS=2'
-# CHECK-CMAKE-PAR-LTO: '-DCMAKE_BUILD_TYPE=Release'
-
-# RUN: env MAX_PARALLEL_TESTS=2 python %{src_root}/zorg/jenkins/monorepo_build.py cmake all | FileCheck --check-prefix CHECK-CMAKE-2-TESTS %s
-# CHECK-CMAKE-2-TESTS: '/usr/local/bin/cmake' '-G' 'Ninja'
-# CHECK-CMAKE-2-TESTS: '-DLLVM_LIT_ARGS=--xunit-xml-output=testresults.xunit.xml -v --timeout=600 -j 2'
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --cmake-type=RelWithDebugInfo | FileCheck --check-prefix CHECK-CMAKE-UPLOADS %s
-# CHECK-CMAKE-UPLOADS: @@@ Uploading Artifact @@@
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py static-analyzer-benchmarks | FileCheck --check-prefix CHECK-STATIC-ANALYZER-BENCHMARKS %s
-# CHECK-STATIC-ANALYZER-BENCHMARKS: @@@ Static Analyzer Benchmarks @@@
-# CHECK-STATIC-ANALYZER-BENCHMARKS: cd [[WORKSPACE:.*]]/test-suite-ClangAnalyzer/
-# CHECK-STATIC-ANALYZER-BENCHMARKS: '[[WORKSPACE]]/utils-analyzer/SATestBuild.py' '--strictness' '0'
-# CHECK-STATIC-ANALYZER-BENCHMARKS: @@@@@@
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --globalisel | FileCheck --check-prefix CHECK-GISEL %s
-# CHECK-GISEL: '/usr/local/bin/cmake' '-G' 'Ninja'
-# CHECK-GISEL: '-DLLVM_BUILD_GLOBAL_ISEL=ON'
-
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --lto --cmake-flag="-DFOO" | FileCheck --check-prefix CHECK-CMAKEFLAGS %s
-# CHECK-CMAKEFLAGS: '-DFOO'
-
-# Make sure you can pass new build targetss.
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake build --cmake-build-target foo --cmake-build-target bar | FileCheck --check-prefix CHECK-BTARGETS %s
-# CHECK-BTARGETS: '/usr/local/bin/ninja' '-v' 'foo' 'bar'
-
-# Make sure you can pass new test targets.
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake test --cmake-test-target foo --cmake-test-target bar | FileCheck --check-prefix CHECK-TTARGETS %s
-# CHECK-TTARGETS: '/usr/local/bin/ninja' '-v' '-k' '0' 'foo' 'bar'
-
-# Test long should always do check-all, since that is what many bots expect.
-# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake testlong | FileCheck --check-prefix CHECK-TTARGETS2 %s
-# CHECK-TTARGETS2: '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all'
Removed: zorg/trunk/zorg/jenkins/monorepo_build.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/jenkins/monorepo_build.py?rev=366296&view=auto
==============================================================================
--- zorg/trunk/zorg/jenkins/monorepo_build.py (original)
+++ zorg/trunk/zorg/jenkins/monorepo_build.py (removed)
@@ -1,914 +0,0 @@
-"""Build and test clangs."""
-
-import sys
-import logging
-import os
-import subprocess
-import datetime
-import time
-import argparse
-import shutil
-import math
-import re
-import xml.etree.ElementTree as ET
-from contextlib import contextmanager
-from urllib2 import urlopen, URLError, HTTPError
-
-SERVER = "labmaster2.lab.llvm.org"
-
-NINJA = "/usr/local/bin/ninja"
-
-# Add dependency checker to the Python path.
-# For relative reference to the dependency file.
-here = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.abspath(here + "/../../dep/"))
-import dep # noqa
-
-
-
-def readme_name(repo):
- """Given a repo, return the name of the readme file."""
- if repo == "libcxx":
- return "LICENSE.TXT"
- return "README.txt"
-
-
-def next_section(name):
- """Jenkins is setup to parse @@@ xyz @@@ as a new section of the buildlog
- with title xyz. The section ends with @@@@@@ """
- footer()
- header(name)
-
-
-def header(name):
- print "@@@", name, "@@@"
-
-
-def footer():
- print "Completed at: " + time.strftime("%FT%T")
- print "@@@@@@"
-
-
-def quote_sh_string(string):
- """Make things that we print shell safe for copy and paste."""
- return "\\'".join("'" + p + "'" for p in string.split("'"))
-
-
-class Configuration(object):
- """docstring for Configuration"""
-
- def __init__(self, args):
- super(Configuration, self).__init__()
- self._args = args
- self.workspace = os.environ.get('WORKSPACE', os.getcwd())
- self._src_dir = os.environ.get('SRC_DIR', 'llvm-project')
- self._llvm_src_dir = os.environ.get('LLVM_SRC_DIR', 'llvm')
- self._lldb_src_dir = os.environ.get('LLDB_SRC_DIR', 'lldb')
- self._build_dir = os.environ.get('BUILD_DIR', 'clang-build')
- self._lldb_build_dir = os.environ.get('LLDB_BUILD_DIR', 'lldb-build')
- self._install_dir = os.environ.get('INSTALL_DIR', 'clang-install')
- self.j_level = os.environ.get('J_LEVEL', None)
- self.max_parallel_tests = os.environ.get('MAX_PARALLEL_TESTS', None)
- self.max_parallel_links = os.environ.get('MAX_PARALLEL_LINKS', None)
- self.host_compiler_url = os.environ.get('HOST_URL',
- 'http://labmaster2.local/artifacts/')
- self.artifact_url = os.environ.get('ARTIFACT', 'NONE')
- self.job_name = os.environ.get('JOB_NAME', 'NONE')
- self.build_id = os.environ.get('BUILD_ID', 'NONE')
- self.build_number = os.environ.get('BUILD_NUMBER', 'NONE')
- self.git_sha = os.environ.get('GIT_SHA', 'NONE')
- self.nobootstrap = True
- self.device = None
- self.node_name = os.environ.get('NODE_NAME', None)
- self.lldb_test_archs = os.environ.get('LLDB_TEST_ARCHS', 'x86_64').split()
-
- # Import all of the command line arguments into the config object
- self.__dict__.update(vars(args))
-
- def builddir(self):
- """The build output directory for this compile."""
- return os.path.join(self.workspace, self._build_dir)
-
- def srcdir(self):
- """The derived source directory for this build."""
- return os.path.join(self.workspace, self._src_dir)
-
- def llvmsrcdir(self):
- """The llvm source directory for this build."""
- return os.path.join(self.workspace, self._src_dir, self._llvm_src_dir)
-
- def lldbbuilddir(self):
- """The derived source directory for this lldb build."""
- return os.path.join(self.workspace, self._lldb_build_dir)
-
- def lldbsrcdir(self):
- """The derived source directory for this lldb build."""
- return os.path.join(self.workspace, self._lldb_src_dir)
-
- def installdir(self):
- """The install directory for the compile."""
- return os.path.join(self.workspace, self._install_dir)
-
- def CC(self):
- """Location of the host compiler, if one is present in this build."""
- cc_basedir = os.path.join(self.workspace, 'host-compiler/')
- if os.path.exists(cc_basedir):
- clang_exec_path = os.path.join(cc_basedir, 'bin/clang')
- assert os.path.exists(clang_exec_path), "host-compiler present," \
- " but has no clang executable."
- return clang_exec_path
- else:
- return False
-
- def liblto(self):
- """Location of the host compiler, if one is present"""
- cc_basedir = os.path.join(self.workspace, 'host-compiler/')
- if os.path.exists(cc_basedir):
- clang_liblto_path = os.path.join(cc_basedir, 'lib/')
- assert os.path.exists(clang_liblto_path), "host-compiler present," \
- " but has no liblto."
- return clang_liblto_path
- else:
- return False
-
- def branch(self):
- """Figure out the source branch name.
- Not using GIT_BRANCH env var from Jenkins as that includes the
- remote name too.
- """
- if not os.environ.get('TESTING', False):
- cmd = ['git', '-C', conf.srcdir(), 'symbolic-ref', '--short', 'HEAD']
- out = run_collect_output(cmd).strip()
- return out
- return 'master'
-
- def link_memory_usage(self):
- """Guesstimate the maximum link memory usage for this build.
- We are only building master here so we will just use that value
- """
- # Determinited experimentally.
- usages = {'master': 3.5}
- if self.branch() == 'master':
- return usages['master']
- else:
- raise NotImplementedError(
- "Unknown link memory usage." + self.branch())
-
-
-# Global storage for configuration object.
-conf = None # type: Configuration
-
-def cmake_builder(target):
- if not os.getenv("TESTING"):
- dep.parse_dependencies([here + "/clang_build_dependencies.dep"])
-
- env = []
- dyld_path = ""
- if conf.lto and conf.liblto():
- dyld_path = conf.liblto()
- env.extend(["env", "DYLD_LIBRARY_PATH=" + dyld_path])
-
- cmake_cmd = env + ["/usr/local/bin/cmake", "-G", "Ninja",
- '-DCMAKE_MAKE_PROGRAM=' + NINJA,
- "-DCMAKE_INSTALL_PREFIX=" + conf.installdir(),
- "-DLLVM_ENABLE_PROJECTS=" + conf.llvm_enable_projects,
- conf.llvmsrcdir()]
-
- compiler_flags = conf.compiler_flags
- max_parallel_links = conf.max_parallel_links
-
- if conf.lto:
- if conf.thinlto:
- cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS=1"]
- else:
- cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS=" + str(max_link_jobs())]
- cmake_cmd += ['-DLLVM_BUILD_EXAMPLES=Off']
- if not max_parallel_links:
- max_parallel_links = 1
- if dyld_path:
- cmake_cmd += ['-DDYLD_LIBRARY_PATH=' + dyld_path]
- else:
- cmake_cmd += ['-DLLVM_ENABLE_LTO=Off']
- cmake_cmd += ['-DLLVM_BUILD_EXAMPLES=On']
-
- cmake_cmd += ["-DCMAKE_MACOSX_RPATH=On"]
-
- libtool_path = query_sys_tool("macosx", "libtool")
- if libtool_path:
- cmake_cmd += ['-DCMAKE_LIBTOOL=' + libtool_path]
-
- if compiler_flags:
- cmake_cmd += ["-DCMAKE_C_FLAGS={}".format(' '.join(compiler_flags)),
- "-DCMAKE_CXX_FLAGS={}".format(' '.join(compiler_flags))]
-
- if max_parallel_links is not None:
- cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS={}".format(max_parallel_links)]
-
- if conf.CC():
- cmake_cmd += ['-DCMAKE_C_COMPILER=' + conf.CC(),
- '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"]
-
- if conf.cmake_build_type:
- cmake_cmd += ["-DCMAKE_BUILD_TYPE=" + conf.cmake_build_type]
- elif conf.debug:
- cmake_cmd += ["-DCMAKE_BUILD_TYPE=Debug"]
- else:
- cmake_cmd += ["-DCMAKE_BUILD_TYPE=Release"]
-
- cmake_cmd += ["-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On"]
-
- for flag in conf.cmake_flags:
- cmake_cmd += [flag]
-
- if conf.assertions:
- cmake_cmd += ["-DLLVM_ENABLE_ASSERTIONS=On"]
- else:
- cmake_cmd += ["-DLLVM_ENABLE_ASSERTIONS=Off"]
-
- if conf.globalisel:
- cmake_cmd += ["-DLLVM_BUILD_GLOBAL_ISEL=ON"]
-
- lit_flags = ['--xunit-xml-output=testresults.xunit.xml', '-v', '--timeout=600']
- if conf.max_parallel_tests:
- lit_flags += ['-j', conf.max_parallel_tests]
- cmake_cmd += ['-DLLVM_LIT_ARGS={}'.format(' '.join(lit_flags))]
-
- ninja_cmd = env + ["/usr/local/bin/ninja", '-v']
- if conf.j_level is not None:
- ninja_cmd += ["-j", conf.j_level]
-
- if target == 'all' or target == 'build':
- header("Cmake")
- run_cmd(conf.builddir(), cmake_cmd)
- footer()
- header("Ninja build")
-
- # Build all if nothing is passed by the user.
- passed_target = conf.cmake_build_targets
- build_target = passed_target if passed_target else ['all']
- run_cmd(conf.builddir(), ninja_cmd + build_target)
- footer()
- if conf.noinstall:
- header("Skip install")
- else:
- header("Ninja install")
- run_cmd(conf.builddir(), ninja_cmd + ['install'])
- build_upload_artifact()
- footer()
- # Run all the test targets.
- ninja_cmd.extend(['-k', '0'])
- if target == 'all' or target == 'test' or target == 'testlong':
- header("Ninja test")
-
- targets = [
- 'check-all'] if target == 'testlong' or target == 'all' else conf.cmake_test_targets
-
- if not targets:
- # testlong and all do check all, otherwise check and check-clang
- # unless the user asked for something else.
- targets = ['check', 'check-clang']
-
- run_cmd(conf.builddir(), ninja_cmd + targets)
- footer()
-
-
-def clang_builder(target):
- """Build to set of commands to compile and test apple-clang"""
-
- # get rid of old archives from prior builds
- run_ws(['sh', '-c', 'rm -rfv *gz'])
-
- if target == "all" or target == "build":
- # Clean the build directory.
- run_ws(['rm', '-rf', 'clang.roots'])
-
- sdk_name = 'macosx'
-
- sdkroot = query_sdk_path(sdk_name)
- libtool_path = query_sys_tool(sdk_name, "libtool")
-
- clang_br = os.path.join(conf.workspace, conf._build_dir)
- next_section("Build Directory")
- run_ws(["mkdir", "-p", clang_br])
-
- toolchain = '/Applications/Xcode.app/Contents/Developer' \
- '/Toolchains/XcodeDefault.xctoolchain'
-
- env = []
- dyld_path = ""
- if conf.lto and conf.liblto():
- dyld_path = conf.liblto()
- env.extend(["env", "DYLD_LIBRARY_PATH=" + dyld_path])
-
- next_section("Build Clang")
- if conf.nobootstrap:
- if conf.debug or conf.device:
- assert False, "Invalid parameter for clang-builder."
- run_cmd(clang_br, ['mkdir',
- './Build',
- './Root'])
- install_prefix = conf.installdir()
-
- # Infer which CMake cache file to use. If ThinLTO we select a specific one.
- cmake_cachefile_thinlto = ''
- if conf.thinlto:
- cmake_cachefile_thinlto = '-ThinLTO'
- cmake_cachefile = '{}/clang/cmake/caches/Apple-stage2{}.cmake'.format(
- conf.srcdir(), cmake_cachefile_thinlto)
-
- cmake_command = env + ["/usr/local/bin/cmake", '-G', 'Ninja', '-C',
- cmake_cachefile,
- '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format(
- "TRUE" if conf.assertions else "FALSE"),
- '-DCMAKE_BUILD_TYPE=RelWithDebInfo',
- '-DLLVM_ENABLE_PROJECTS={}'.format(conf.llvm_enable_projects),
- '-DCMAKE_MAKE_PROGRAM=' + NINJA,
- '-DLLVM_VERSION_PATCH=99',
- '-DLLVM_VERSION_SUFFIX=""',
- '-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On',
- '-DCLANG_COMPILER_RT_CMAKE_ARGS={}/compiler-rt/cmake/caches/Apple.cmake'.format(
- conf.srcdir()),
- '-DCOMPILER_RT_BUILD_SANITIZERS=On',
- '-DCMAKE_INSTALL_PREFIX={}'.format(
- install_prefix),
- '-DCLANG_REPOSITORY_STRING={}'.format(
- conf.branch()),
- '-DCLANG_APPEND_VC_REV=On',
- '-DLLVM_BUILD_TESTS=On',
- '-DLLVM_INCLUDE_TESTS=On',
- '-DCLANG_INCLUDE_TESTS=On',
- '-DLLVM_INCLUDE_UTILS=On',
- '-DLIBCXX_INSTALL_HEADERS=On',
- '-DLIBCXX_OVERRIDE_DARWIN_INSTALL=On',
- '-DLIBCXX_INSTALL_LIBRARY=Off',
- '-DCMAKE_MACOSX_RPATH=On',
- ]
-
- if dyld_path:
- cmake_command += ['-DDYLD_LIBRARY_PATH=' + dyld_path]
-
- if libtool_path:
- cmake_command += ['-DCMAKE_LIBTOOL=' + libtool_path]
-
- if conf.CC():
- cmake_command.extend(['-DCMAKE_C_COMPILER=' + conf.CC(),
- '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"])
-
- lit_flags = ['--xunit-xml-output=testresults.xunit.xml', '-v', '--timeout=600']
- if conf.max_parallel_tests:
- lit_flags += ['-j', conf.max_parallel_tests]
- cmake_command.extend(
- ['-DLLVM_LIT_ARGS={}'.format(' '.join(lit_flags))])
-
- if conf.thinlto:
- cmake_command.extend(["-DLLVM_PARALLEL_LINK_JOBS=1"])
- elif conf.lto:
- cmake_command.extend(
- ["-DLLVM_PARALLEL_LINK_JOBS=" + str(max_link_jobs())])
- else:
- cmake_command.extend(['-DLLVM_ENABLE_LTO=Off'])
- cmake_command.extend([
- '-DCMAKE_C_FLAGS_RELWITHDEBINFO:STRING=-O2 -gline-tables-only -DNDEBUG',
- '-DCMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -gline-tables-only -DNDEBUG'])
-
- for flag in conf.cmake_flags:
- cmake_command += [flag]
-
- cmake_command.append(conf.llvmsrcdir())
- run_cmd(os.path.join(clang_br, 'Build'), cmake_command)
- next_section("Ninja")
- run_cmd(os.path.join(clang_br, 'Build'), [NINJA, '-v', 'install'])
-
- build_upload_artifact()
-
- else:
- # Two stage build, via the make files.
- print 'Stage two compile TBD in near future'
-
- if not conf.device and (target == "test" or target == "all"):
- # Add steps to run the tests.
- next_section("Tests")
- # Auto detect bootstrap and non-bootstrap.
- obj_dir = os.path.join(conf._build_dir,
- 'Objects/obj-llvm/tools/clang/stage2-bins/')
- if not os.path.exists(obj_dir):
- obj_dir = os.path.join(conf._build_dir, 'Build/')
- obj_dir = os.path.join(conf.workspace, obj_dir)
-
- cmd = [NINJA, '-v', '-k', '0', 'check-all']
-
- if conf.assertions:
- cmd[-1] += ' --param use_gmalloc=1 ' \
- '--param gmalloc_path=$(xcodebuild -find-library' \
- ' libgmalloc.dylib)'
- run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'})
-
-
-def parse_settings_from_output(working_dir, cmd):
- old_dir = os.getcwd()
- try:
- os.chdir(working_dir)
- assignment_regex = re.compile(r"^\s+([^\s=]+)\s*=\s*(.+)$")
- settings = {}
- for line in subprocess.check_output(cmd).splitlines(True):
- match = assignment_regex.match(line)
- if match:
- settings[match.group(1)] = match.group(2)
- return settings
- finally:
- os.chdir(old_dir)
-
-
-def lldb_builder():
- """Do an Xcode build of lldb."""
-
- # Wipe the build folder
-
- header("Clean LLDB build directory")
- if os.path.exists(conf.lldbbuilddir()):
- shutil.rmtree(conf.lldbbuilddir())
- footer()
-
- # Build into the build folder
- build_configuration = "Release"
-
- xcodebuild_cmd = [
- "xcodebuild",
- "-arch", "x86_64",
- "-configuration", build_configuration,
- "-scheme", "desktop",
- "-derivedDataPath", conf.lldbbuilddir()
- # It is too fragile to use the Xcode debugserver. If we add new
- # command line arguments to debugserver, the older Xcode debugserver
- # will fall over and not run. By commenting out this flag, we
- # are requiring the builder to have the lldb_codesign code signing
- # certificate and we are ensuring we are testing the latest debugserver
- # from lldb.
- # "DEBUGSERVER_USE_FROM_SYSTEM=1"
- ]
-
- header("Build Xcode desktop scheme")
- run_cmd("lldb", xcodebuild_cmd)
- footer()
-
- header("Gather Xcode build settings")
- xcodebuild_cmd.append("-showBuildSettings")
- settings = parse_settings_from_output("lldb", xcodebuild_cmd)
- footer()
-
- build_dir = settings.get("BUILD_DIR", None)
- built_products_dir = settings.get("BUILT_PRODUCTS_DIR", None)
- if build_dir is None or built_products_dir is None:
- raise Exception("failed to retrieve build-related directories "
- "from Xcode")
-
- llvm_build_dir = settings.get("LLVM_BUILD_DIR", None)
- llvm_build_dir_arch = settings.get("LLVM_BUILD_DIR_ARCH", None)
- if llvm_build_dir is None or llvm_build_dir_arch is None:
- raise Exception("failed to retrieve LLVM build-related settings "
- "from Xcode")
- llvm_build_bin_dir = os.path.join(llvm_build_dir, llvm_build_dir_arch, "bin")
- built_clang_path = os.path.join(llvm_build_bin_dir, "clang")
- built_filecheck_path = os.path.join(llvm_build_bin_dir, "FileCheck")
- effective_clang = os.environ.get("LLDB_PYTHON_TESTSUITE_CC",
- built_clang_path)
-
- # Run C++ test suite (gtests)
-
- xcodebuild_cmd = [
- "xcodebuild",
- "-arch", "x86_64",
- "-configuration", build_configuration,
- "-scheme", "lldb-gtest",
- "-derivedDataPath", conf.lldbbuilddir(),
- # See notes above.
- # "DEBUGSERVER_USE_FROM_SYSTEM=1"
- ]
-
- header("Build Xcode lldb-gtest scheme")
- run_cmd("lldb", xcodebuild_cmd)
- footer()
-
- # Run LLDB Python test suite for archs defined in LLDB_TEST_ARCHS
- for arch in conf.lldb_test_archs:
- results_file = os.path.join(build_dir,
- "test-results-{}.xml".format(arch))
- python_testsuite_cmd = [
- "/usr/bin/python",
- "test/dotest.py",
- "--executable", os.path.join(built_products_dir, "lldb"),
- "-C", effective_clang,
- "--arch", arch,
- "--results-formatter",
- "lldbsuite.test_event.formatter.xunit.XunitFormatter",
- "--results-file", results_file,
- "--rerun-all-issues",
- "--env", "TERM=vt100",
- "-O--xpass=ignore",
- "--dsymutil="+os.path.join(os.path.dirname(effective_clang), 'dsymutil'),
- "--filecheck="+built_filecheck_path
- ]
-
- header("Run LLDB Python-based test suite ({} targets)".format(arch))
- # For the unit tests, we don't want to stop the build if there are
- # build errors. We allow the JUnit/xUnit parser to pick this up.
- print repr(python_testsuite_cmd)
- run_cmd_errors_okay("lldb", python_testsuite_cmd)
- footer()
-
-
-def lldb_cmake_builder():
- """Do a CMake build of lldb."""
-
- test_dir = os.path.join(conf.workspace, 'test')
- log_dir = os.path.join(test_dir, 'logs')
- results_file = os.path.join(test_dir, 'results.xml')
- dest_dir = os.path.join(conf.workspace, 'results', 'lldb')
- run_ws(["mkdir", "-p", conf.lldbbuilddir()])
- cmake_build_type = conf.cmake_build_type if conf.cmake_build_type else 'RelWithDebInfo'
- header("Configure")
- dotest_args=['--arch', 'x86_64', '--build-dir',
- conf.lldbbuilddir()+'/lldb-test-build.noindex',
- '-s='+log_dir,
- '-t',
- '--env', 'TERM=vt100']
- dotest_args.extend(conf.dotest_flags)
- cmake_cmd = ["/usr/local/bin/cmake", '-G', 'Ninja',
- conf.srcdir(),
- '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format(
- "TRUE" if conf.assertions else "FALSE"),
- '-DLLVM_ENABLE_PROJECTS='+conf.llvm_enable_projects,
- '-DCMAKE_BUILD_TYPE='+cmake_build_type,
- '-DCMAKE_MAKE_PROGRAM=' + NINJA,
- '-DLLVM_VERSION_PATCH=99',
- '-DLLVM_ENABLE_MODULES=On',
- '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
- '-DCMAKE_INSTALL_PREFIX="%s"'%dest_dir,
- '-DLLDB_TEST_USER_ARGS='+';'.join(dotest_args),
- '-DLLVM_LIT_ARGS=--xunit-xml-output=%s -v'%results_file]
- cmake_cmd.extend(conf.cmake_flags)
-
- if conf.CC():
- cmake_cmd.extend(['-DCMAKE_C_COMPILER=' + conf.CC(),
- '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"])
-
- run_cmd(conf.lldbbuilddir(), cmake_cmd)
- footer()
-
- header("Build")
- run_cmd(conf.lldbbuilddir(), [NINJA, '-v'])
- footer()
-
- header("Run Tests")
- run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo'])
- run_cmd(conf.lldbbuilddir(), ['/usr/bin/env', 'TERM=vt100', NINJA, '-v',
- 'check-lldb'])
- footer()
-
-
-def static_analyzer_benchmarks_builder():
- """Run static analyzer benchmarks"""
- header("Static Analyzer Benchmarks")
-
- benchmark_script = conf.workspace + "/utils-analyzer/SATestBuild.py"
- benchmarks_dir = conf.workspace + "/test-suite-ClangAnalyzer/"
-
- compiler_bin_dir = conf.workspace + "/host-compiler/bin/"
- scanbuild_bin_dir = conf.workspace + "/tools-scan-build/bin/"
-
- old_path = os.environ.get("PATH", "")
- env = dict(os.environ, PATH=compiler_bin_dir + os.pathsep +
- scanbuild_bin_dir + os.pathsep +
- old_path)
-
- benchmark_cmd = [benchmark_script,
- "--strictness", "0"
- ]
- run_cmd(benchmarks_dir, benchmark_cmd, env=env)
-
- footer()
-
-
-def http_download(url, dest):
- """Safely download url to dest.
-
- Print error and exit if download fails.
- """
- try:
- print "GETting", url, "to", dest, "...",
- f = urlopen(url)
- # Open our local file for writing
- with open(dest, "wb") as local_file:
- local_file.write(f.read())
-
- except HTTPError, e:
- print
- print "HTTP Error:", e.code, url
- sys.exit(1)
-
- except URLError, e:
- print
- print "URL Error:", e.reason, url
- sys.exit(1)
- print "done."
-
-
-def create_builddirs():
- paths = [conf.builddir(), conf.installdir()]
- for p in paths:
- if not os.path.exists(p):
- os.makedirs(p)
-
-
-def fetch_compiler():
- local_name = "host-compiler.tar.gz"
- url = conf.host_compiler_url + "/" + conf.artifact_url
- header("Fetching Compiler")
- http_download(url, conf.workspace + "/" + local_name)
- print "Decompressing..."
- if os.path.exists(conf.workspace + "/host-compiler"):
- shutil.rmtree(conf.workspace + "/host-compiler")
- os.mkdir(conf.workspace + "/host-compiler")
- run_cmd(conf.workspace + "/host-compiler/",
- ['tar', 'zxf', "../" + local_name])
- os.unlink(local_name)
- footer()
-
-def build_upload_artifact():
- """Create artifact for this build, and upload to server."""
- if conf.noupload:
- print 'Not uploading artificats'
- return
- header("Uploading Artifact")
- prop_file = "last_good_build.properties"
-
- # TODO : azhar : Update artifact name
- artifact_name = "clang-t{}-b{}.tar.gz".format(conf.build_id,
- conf.build_number)
- new_url = conf.job_name + "/" + artifact_name
-
- with open(prop_file, 'w') as prop_fd:
- prop_fd.write("GIT_SHA={}\n".format(conf.git_sha))
- prop_fd.write("ARTIFACT={}\n".format(new_url))
-
- # The .a's are big and we don't need them later. Drop the LLVM and clang
- # libraries, but keep the libraries from compiler-rt.
- tar = ["tar", "zcvf", "../" + artifact_name, "--exclude=*libLLVM*.a",
- "--exclude=*libclang[A-Z]*.a", "."]
-
- run_cmd(conf.installdir(), tar)
-
- mkdir_cmd = ["ssh", "buildslave@" + SERVER, "mkdir", "-p", "/Library/WebServer/Documents/artifacts/" + conf.job_name]
-
- run_cmd(conf.workspace, mkdir_cmd)
-
- upload_cmd = ["scp", artifact_name,
- "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" +
- conf.job_name + "/"]
-
- run_cmd(conf.workspace, upload_cmd)
-
- upload_cmd = ["scp", prop_file,
- "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" +
- conf.job_name + "/"]
-
- run_cmd(conf.workspace, upload_cmd)
-
- ln_cmd = ["ssh", "buildslave@" + SERVER,
- "ln", "-fs", "/Library/WebServer/Documents/artifacts/" +
- conf.job_name + "/" + artifact_name,
- "/Library/WebServer/Documents/artifacts/" +
- conf.job_name + "/latest"]
-
- run_cmd(conf.workspace, ln_cmd)
-
- lng_cmd = ["ssh", "buildslave@" + SERVER,
- "ln", "-fs", "/Library/WebServer/Documents/artifacts/" +
- conf.job_name + "/" + artifact_name,
- "/Library/WebServer/Documents/artifacts/" +
- conf.job_name + "/g" + conf.git_sha]
- run_cmd(conf.workspace, lng_cmd)
-
-
-def run_cmd(working_dir, cmd, env=None, sudo=False, err_okay=False):
- """Run a command in a working directory, and make sure it returns zero."""
- assert type(cmd) == list, "Not a list: {}".format(type(cmd))
- old_cwd = os.getcwd()
- if env:
- envs = []
- for key, value in env.items():
- envs.append("{}={}".format(key, value))
- cmd = ["env"] + envs + cmd
- if sudo:
- cmd = ['sudo'] + cmd
-
- cmd_to_print = ' '.join([quote_sh_string(x) for x in cmd])
- sys.stdout.write("cd {}\n{}\n".format(working_dir, cmd_to_print))
- sys.stdout.flush()
- return_code = 0
- start_time = datetime.datetime.now()
- if not os.environ.get('TESTING', False):
- try:
- os.chdir(working_dir)
- subprocess.check_call(cmd)
- os.chdir(old_cwd)
- except subprocess.CalledProcessError as excpt:
- if not err_okay:
- raise excpt
- else:
- logging.info("Ignoring failed command.")
- return_code = excpt.returncode
- end_time = datetime.datetime.now()
-
- logging.info("Command took {} seconds".format(
- (end_time - start_time).seconds))
- return return_code
-
-
-def run_cmd_errors_okay(working_dir, cmd, env=None):
- """Run a command in a working directory, reporting return value.
- Non-zero exit codes do not generate an exception.
- """
- old_cwd = os.getcwd()
- cmd_to_print = ' '.join([quote_sh_string(x) for x in cmd])
- sys.stdout.write("cd {}\n{}\n".format(working_dir, cmd_to_print))
- sys.stdout.flush()
-
- start_time = datetime.datetime.now()
- if not os.environ.get('TESTING', False):
- try:
- os.chdir(working_dir)
- result = subprocess.call(cmd, env=env)
- finally:
- os.chdir(old_cwd)
- end_time = datetime.datetime.now()
-
- logging.info("Command took {} seconds: return code {}".format(
- (end_time - start_time).seconds, result))
-
-
-KNOWN_TARGETS = ['all', 'build', 'test', 'testlong']
-KNOWN_BUILDS = ['clang', 'cmake', 'lldb', 'lldb-cmake', 'fetch', 'artifact',
- 'static-analyzer-benchmarks']
-
-
-def query_sdk_path(sdk_name):
- """Get the path to the sdk named using xcrun.
-
- When $TESTING define, just give a dummy back. We do this because xcrun
- could fail if the sdk you want is not installed, and that is silly for
- testing.
- """
-
- if not os.environ.get('TESTING', False):
- cmd = ['xcrun', '--sdk', sdk_name, '--show-sdk-path']
- return run_collect_output(cmd).strip()
- else:
- return "/Applications/Xcode.app/Contents/Developer/Platforms/" \
- "MacOSX.platform/Developer/SDKs/MacOSX10.10.sdk"
-
-
-def max_link_jobs():
- """Link jobs take about 3.6GB of memory, max."""
- mem_str = run_collect_output(["sysctl", "hw.memsize"])
- mem = float(mem_str.split()[1].strip())
- mem = mem / (1024.0 ** 3) # Conver to GBs.
- return int(math.ceil(mem / conf.link_memory_usage()))
-
-
-TEST_VALS = {"sysctl hw.ncpu": "hw.ncpu: 8\n",
- "sysctl hw.memsize": "hw.memsize: 8589934592\n",
- "xcrun --sdk iphoneos --show-sdk-path": "/Foo/bar",
- "/usr/bin/xcrun svn upgrade": "",
- }
-
-
- at contextmanager
-def cwd(path):
- last_cwd = os.getcwd()
- if path:
- os.chdir(path)
- try:
- yield
- finally:
- os.chdir(last_cwd)
-
-
-def run_collect_output(cmd, working_dir=None, stderr=None):
- """Run cmd and then return the output.
-
- If working_dir is supplied the cmd will run in
- with a context manager in working_dir.
- """
- if os.getenv("TESTING"):
- print 'TV: ' + ' '.join(cmd)
- return TEST_VALS[' '.join(cmd)]
-
- with cwd(working_dir):
- return subprocess.check_output(cmd, stderr=stderr)
-
-
-def query_sys_tool(sdk_name, tool_name):
- """Get the path of system tool
-
- When $TESTING define, just give a dummy back.
- """
-
- if not os.environ.get('TESTING', False):
- cmd = ['xcrun', '--sdk', sdk_name, '--find', tool_name]
- return run_collect_output(cmd).strip()
- else:
- return "/usr/bin/" + tool_name
-
-
-def run_ws(cmd, env=None):
- """Wrapper to call run_cmd in local workspace.
-
- Since 99 percent of the time, that is where you want to call things from.
- """
- return run_cmd(conf.workspace, cmd, env)
-
-
-def parse_args():
- """Get the command line arguments, and make sure they are correct."""
-
- parser = argparse.ArgumentParser(
- description='Build and test compilers and other things.')
-
- parser.add_argument("build_type",
- help="The kind of build to trigger.",
- choices=KNOWN_BUILDS)
-
- parser.add_argument("build_target",
- nargs='?',
- help="The targets to call (build, check, etc).",
- choices=KNOWN_TARGETS)
-
- parser.add_argument('--assertions', dest='assertions', action='store_true')
- parser.add_argument('--lto', dest='lto', action='store_true')
- parser.add_argument('--thinlto', dest='thinlto', action='store_true')
- parser.add_argument('--debug', dest='debug', action='store_true')
- parser.add_argument('--cmake-type', dest='cmake_build_type',
- help="Override cmake type Release, Debug, "
- "RelWithDebInfo and MinSizeRel")
- parser.add_argument('--cmake-flag', dest='cmake_flags',
- action='append', default=[],
- help='Set an arbitrary cmake flag')
- parser.add_argument('--dotest-flag', dest='dotest_flags',
- action='append', default=[],
- help='Set an arbitrary lldb dotest.py flag')
- parser.add_argument('--cmake-test-target', dest='cmake_test_targets',
- action='append', default=[],
- help='Targets to build during testing')
- parser.add_argument('--cmake-build-target', dest='cmake_build_targets',
- action='append', default=[],
- help='Targets to build during building.')
- parser.add_argument('--compiler-flag', dest='compiler_flags',
- action='append', default=[],
- help='Set an arbitrary compiler flag')
- parser.add_argument('--noupload', dest='noupload', action='store_true')
- parser.add_argument('--noinstall', dest='noinstall', action='store_true',
- help="Disable the install stage, build only.")
- parser.add_argument('--globalisel', dest='globalisel',
- action='store_true', help="Turn on the experimental"
- " GlobalISel CMake flag.")
- parser.add_argument('--projects', dest='llvm_enable_projects',
- default="clang;clang-tools-extra;compiler-rt;libcxx",
- help="Semicolon seperated list of projects to build.")
-
- args = parser.parse_args()
- if args.thinlto:
- args.lto = True
- return args
-
-
-def main():
- """Run a build based on command line args and ENV."""
- global conf
- args = parse_args()
- conf = Configuration(args)
-
- create_builddirs()
- try:
- if args.build_type == 'clang':
- clang_builder(args.build_target)
- elif args.build_type == 'lldb':
- lldb_builder()
- elif args.build_type == 'lldb-cmake':
- lldb_cmake_builder()
- elif args.build_type == 'cmake':
- cmake_builder(args.build_target)
- elif args.build_type == 'fetch':
- fetch_compiler()
- elif args.build_type == 'artifact':
- build_upload_artifact()
- elif args.build_type == 'static-analyzer-benchmarks':
- static_analyzer_benchmarks_builder()
- except subprocess.CalledProcessError as exct:
- print "Command failed", exct.message
- print "Command:", exct.cmd
- sys.exit(1)
-
-
-if __name__ == '__main__':
- main()
More information about the llvm-commits
mailing list