[zorg] r365440 - [zorg] Add a new build script for use with monorepo
Azharuddin Mohammed via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 8 22:23:22 PDT 2019
Author: azharudd
Date: Mon Jul 8 22:23:22 2019
New Revision: 365440
URL: http://llvm.org/viewvc/llvm-project?rev=365440&view=rev
Log:
[zorg] Add a new build script for use with monorepo
Summary:
monorepo-build.py is similar in functionality to the currently build.py, but
with changes to make it work with the monorepo.
Reviewers: cmatthews
Reviewed By: cmatthews
Subscribers: llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D64028
Added:
zorg/trunk/test/jenkins/test_monorepo_build.py
zorg/trunk/zorg/jenkins/monorepo_build.py
Added: zorg/trunk/test/jenkins/test_monorepo_build.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/test/jenkins/test_monorepo_build.py?rev=365440&view=auto
==============================================================================
--- zorg/trunk/test/jenkins/test_monorepo_build.py (added)
+++ zorg/trunk/test/jenkins/test_monorepo_build.py Mon Jul 8 22:23:22 2019
@@ -0,0 +1,132 @@
+# Testing for the Jenkins monorepo_build.py script
+#
+# RUN: export TESTING=1
+# RUN: export JOB_NAME="FOO"
+# RUN: export BUILD_NUMBER=321
+# RUN: export BRANCH=master
+# Tell monorepo_build.py to just print commands instead of running.
+# RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/libcxx %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb
+# RUN: touch %t.SANDBOX/host-compiler/bin/clang
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all > %t.log
+# RUN: FileCheck --check-prefix CHECK-SIMPLE < %t.log %s
+# CHECK-SIMPLE: @@@ Build Directory @@@
+# CHECK-SIMPLE: cd
+# CHECK-SIMPLE: 'mkdir' '-p'
+# CHECK-SIMPLE: @@@@@@
+# CHECK-SIMPLE: @@@ Build Clang @@@
+# CHECK-SIMPLE: cd
+# CHECK-SIMPLE: 'mkdir' './Build' './Root'
+# CHECK-SIMPLE: cd
+# CHECK-SIMPLE: '/usr/local/bin/cmake' '-G' 'Ninja' '-C'
+# CHECK-SIMPLE: '-DLLVM_ENABLE_ASSERTIONS:BOOL=FALSE'
+# CHECK-SIMPLE: '-DCMAKE_BUILD_TYPE=RelWithDebInfo'
+# CHECK-SIMPLE: '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;compiler-rt;libcxx'
+# CHECK-SIMPLE: '-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja'
+# CHECK-SIMPLE: '-DLLVM_VERSION_PATCH=99'
+# CHECK-SIMPLE: '-DLLVM_VERSION_SUFFIX=""'
+# CHECK-SIMPLE: '-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On'
+# CHECK-SIMPLE: '-DCLANG_COMPILER_RT_CMAKE_ARGS
+# CHECK-SIMPLE: Apple.cmake'
+# CHECK-SIMPLE: '-DCOMPILER_RT_BUILD_SANITIZERS=On'
+# CHECK-SIMPLE: '-DCMAKE_INSTALL_PREFIX
+# CHECK-SIMPLE: '-DCLANG_APPEND_VC_REV=On'
+# CHECK-SIMPLE: '-DLLVM_BUILD_TESTS=On'
+# CHECK-SIMPLE: '-DLLVM_INCLUDE_TESTS=On'
+# CHECK-SIMPLE: '-DCLANG_INCLUDE_TESTS=On'
+# CHECK-SIMPLE: '-DLLVM_INCLUDE_UTILS=On'
+# CHECK-SIMPLE: '-DCMAKE_MACOSX_RPATH=On'
+# CHECK-SIMPLE: '-DLLVM_ENABLE_LTO=Off
+# CHECK-SIMPLE-NOT: -DLLVM_PARALLEL_LINK_JOBS
+# CHECK-SIMPLE: @@@@@@
+# CHECK-SIMPLE: @@@ Ninja @@@
+# CHECK-SIMPLE: cd
+# CHECK-SIMPLE: '/usr/local/bin/ninja' '-v' 'install'
+# CHECK-SIMPLE: @@@@@@
+# CHECK-SIMPLE: @@@ Tests @@@
+
+# CHECK-SIMPLE: cd
+# CHECK-SIMPLE: 'env' 'MALLOC_LOG_FILE=/dev/null' '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all'
+
+# Now Check Assertion Builds have --enable assertions
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --assertions > %t-assert.log
+# RUN: FileCheck --check-prefix CHECK-ASSERT < %t-assert.log %s
+# CHECK-ASSERT: '/usr/local/bin/cmake' '-G' 'Ninja' '-C'
+# CHECK-ASSERT: '-DLLVM_ENABLE_ASSERTIONS:BOOL=TRUE'
+
+# Check LTO
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --lto > %t-lto.log
+# RUN: FileCheck --check-prefix CHECK-LTO < %t-lto.log %s
+# CHECK-LTO: -DLLVM_PARALLEL_LINK_JOBS
+# CHECK-LTO-NOT:: '-DLLVM_ENABLE_LTO=Off
+
+# Now try just a build
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang build --lto
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang build
+
+# Just a test
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang test
+
+# CMake
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --debug > %t-cmake.log
+# RUN: FileCheck --check-prefix CHECK-CMAKE < %t-cmake.log %s
+# CHECK-CMAKE: '/usr/local/bin/cmake' '-G' 'Ninja'
+# CHECK-CMAKE: -DLLVM_BUILD_EXAMPLES=On
+# CHECK-CMAKE: '-DCMAKE_BUILD_TYPE=Debug'
+# CHECK-CMAKE: '-DLLVM_ENABLE_ASSERTIONS=Off'
+# CHECK-CMAKE: -DLLVM_LIT_ARGS=--xunit-xml-output=testresults.xunit.xml -v --timeout=600
+# CHECK-CMAKE: '/usr/local/bin/ninja' '-v' 'all'
+# CHECK-CMAKE: '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all'
+
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake build
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake test
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake testlong
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --lto | FileCheck --check-prefix CHECK-CMAKELTO %s
+# CHECK-CMAKELTO: '/usr/local/bin/cmake' '-G' 'Ninja'
+# CHECK-CMAKELTO: '-DLLVM_BUILD_EXAMPLES=Off'
+# CHECK-CMAKELTO-NOT:: '-DLLVM_ENABLE_LTO=Off
+# CHECK-CMAKELTO: '-DLLVM_PARALLEL_LINK_JOBS=1'
+# CHECK-CMAKELTO: '-DCMAKE_BUILD_TYPE=Release'
+
+# RUN: env MAX_PARALLEL_LINKS=2 python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --lto | FileCheck --check-prefix CHECK-CMAKE-PAR-LTO %s
+# CHECK-CMAKE-PAR-LTO: '/usr/local/bin/cmake' '-G' 'Ninja'
+# CHECK-CMAKE-PAR-LTO: '-DLLVM_BUILD_EXAMPLES=Off'
+# CHECK-CMAKE-PAR-LTO-NOT:: '-DLLVM_ENABLE_LTO=Off
+# CHECK-CMAKE-PAR-LTO: '-DLLVM_PARALLEL_LINK_JOBS=2'
+# CHECK-CMAKE-PAR-LTO: '-DCMAKE_BUILD_TYPE=Release'
+
+# RUN: env MAX_PARALLEL_TESTS=2 python %{src_root}/zorg/jenkins/monorepo_build.py cmake all | FileCheck --check-prefix CHECK-CMAKE-2-TESTS %s
+# CHECK-CMAKE-2-TESTS: '/usr/local/bin/cmake' '-G' 'Ninja'
+# CHECK-CMAKE-2-TESTS: '-DLLVM_LIT_ARGS=--xunit-xml-output=testresults.xunit.xml -v --timeout=600 -j 2'
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --cmake-type=RelWithDebugInfo | FileCheck --check-prefix CHECK-CMAKE-UPLOADS %s
+# CHECK-CMAKE-UPLOADS: @@@ Uploading Artifact @@@
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py static-analyzer-benchmarks | FileCheck --check-prefix CHECK-STATIC-ANALYZER-BENCHMARKS %s
+# CHECK-STATIC-ANALYZER-BENCHMARKS: @@@ Static Analyzer Benchmarks @@@
+# CHECK-STATIC-ANALYZER-BENCHMARKS: cd [[WORKSPACE:.*]]/test-suite-ClangAnalyzer/
+# CHECK-STATIC-ANALYZER-BENCHMARKS: '[[WORKSPACE]]/utils-analyzer/SATestBuild.py' '--strictness' '0'
+# CHECK-STATIC-ANALYZER-BENCHMARKS: @@@@@@
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --globalisel | FileCheck --check-prefix CHECK-GISEL %s
+# CHECK-GISEL: '/usr/local/bin/cmake' '-G' 'Ninja'
+# CHECK-GISEL: '-DLLVM_BUILD_GLOBAL_ISEL=ON'
+
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --lto --cmake-flag="-DFOO" | FileCheck --check-prefix CHECK-CMAKEFLAGS %s
+# CHECK-CMAKEFLAGS: '-DFOO'
+
+# Make sure you can pass new build targetss.
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake build --cmake-build-target foo --cmake-build-target bar | FileCheck --check-prefix CHECK-BTARGETS %s
+# CHECK-BTARGETS: '/usr/local/bin/ninja' '-v' 'foo' 'bar'
+
+# Make sure you can pass new test targets.
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake test --cmake-test-target foo --cmake-test-target bar | FileCheck --check-prefix CHECK-TTARGETS %s
+# CHECK-TTARGETS: '/usr/local/bin/ninja' '-v' '-k' '0' 'foo' 'bar'
+
+# Test long should always do check-all, since that is what many bots expect.
+# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake testlong | FileCheck --check-prefix CHECK-TTARGETS2 %s
+# CHECK-TTARGETS2: '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all'
Added: zorg/trunk/zorg/jenkins/monorepo_build.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/jenkins/monorepo_build.py?rev=365440&view=auto
==============================================================================
--- zorg/trunk/zorg/jenkins/monorepo_build.py (added)
+++ zorg/trunk/zorg/jenkins/monorepo_build.py Mon Jul 8 22:23:22 2019
@@ -0,0 +1,914 @@
+"""Build and test clangs."""
+
+import sys
+import logging
+import os
+import subprocess
+import datetime
+import time
+import argparse
+import shutil
+import math
+import re
+import xml.etree.ElementTree as ET
+from contextlib import contextmanager
+from urllib2 import urlopen, URLError, HTTPError
+
+SERVER = "labmaster2.lab.llvm.org"
+
+NINJA = "/usr/local/bin/ninja"
+
+# Add dependency checker to the Python path.
+# For relative reference to the dependency file.
+here = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.abspath(here + "/../../dep/"))
+import dep # noqa
+
+
+
+def readme_name(repo):
+ """Given a repo, return the name of the readme file."""
+ if repo == "libcxx":
+ return "LICENSE.TXT"
+ return "README.txt"
+
+
+def next_section(name):
+ """Jenkins is setup to parse @@@ xyz @@@ as a new section of the buildlog
+ with title xyz. The section ends with @@@@@@ """
+ footer()
+ header(name)
+
+
+def header(name):
+ print "@@@", name, "@@@"
+
+
+def footer():
+ print "Completed at: " + time.strftime("%FT%T")
+ print "@@@@@@"
+
+
+def quote_sh_string(string):
+ """Make things that we print shell safe for copy and paste."""
+ return "\\'".join("'" + p + "'" for p in string.split("'"))
+
+
+class Configuration(object):
+ """docstring for Configuration"""
+
+ def __init__(self, args):
+ super(Configuration, self).__init__()
+ self._args = args
+ self.workspace = os.environ.get('WORKSPACE', os.getcwd())
+ self._src_dir = os.environ.get('SRC_DIR', 'llvm-project')
+ self._llvm_src_dir = os.environ.get('LLVM_SRC_DIR', 'llvm')
+ self._lldb_src_dir = os.environ.get('LLDB_SRC_DIR', 'lldb')
+ self._build_dir = os.environ.get('BUILD_DIR', 'clang-build')
+ self._lldb_build_dir = os.environ.get('LLDB_BUILD_DIR', 'lldb-build')
+ self._install_dir = os.environ.get('INSTALL_DIR', 'clang-install')
+ self.j_level = os.environ.get('J_LEVEL', None)
+ self.max_parallel_tests = os.environ.get('MAX_PARALLEL_TESTS', None)
+ self.max_parallel_links = os.environ.get('MAX_PARALLEL_LINKS', None)
+ self.host_compiler_url = os.environ.get('HOST_URL',
+ 'http://labmaster2.local/artifacts/')
+ self.artifact_url = os.environ.get('ARTIFACT', 'NONE')
+ self.job_name = os.environ.get('JOB_NAME', 'NONE')
+ self.build_id = os.environ.get('BUILD_ID', 'NONE')
+ self.build_number = os.environ.get('BUILD_NUMBER', 'NONE')
+ self.git_sha = os.environ.get('GIT_SHA', 'NONE')
+ self.nobootstrap = True
+ self.device = None
+ self.node_name = os.environ.get('NODE_NAME', None)
+ self.lldb_test_archs = os.environ.get('LLDB_TEST_ARCHS', 'x86_64').split()
+
+ # Import all of the command line arguments into the config object
+ self.__dict__.update(vars(args))
+
+ def builddir(self):
+ """The build output directory for this compile."""
+ return os.path.join(self.workspace, self._build_dir)
+
+ def srcdir(self):
+ """The derived source directory for this build."""
+ return os.path.join(self.workspace, self._src_dir)
+
+ def llvmsrcdir(self):
+ """The llvm source directory for this build."""
+ return os.path.join(self.workspace, self._src_dir, self._llvm_src_dir)
+
+ def lldbbuilddir(self):
+ """The derived source directory for this lldb build."""
+ return os.path.join(self.workspace, self._lldb_build_dir)
+
+ def lldbsrcdir(self):
+ """The derived source directory for this lldb build."""
+ return os.path.join(self.workspace, self._lldb_src_dir)
+
+ def installdir(self):
+ """The install directory for the compile."""
+ return os.path.join(self.workspace, self._install_dir)
+
+ def CC(self):
+ """Location of the host compiler, if one is present in this build."""
+ cc_basedir = os.path.join(self.workspace, 'host-compiler/')
+ if os.path.exists(cc_basedir):
+ clang_exec_path = os.path.join(cc_basedir, 'bin/clang')
+ assert os.path.exists(clang_exec_path), "host-compiler present," \
+ " but has no clang executable."
+ return clang_exec_path
+ else:
+ return False
+
+ def liblto(self):
+ """Location of the host compiler, if one is present"""
+ cc_basedir = os.path.join(self.workspace, 'host-compiler/')
+ if os.path.exists(cc_basedir):
+ clang_liblto_path = os.path.join(cc_basedir, 'lib/')
+ assert os.path.exists(clang_liblto_path), "host-compiler present," \
+ " but has no liblto."
+ return clang_liblto_path
+ else:
+ return False
+
+ def branch(self):
+ """Figure out the source branch name.
+ Not using GIT_BRANCH env var from Jenkins as that includes the
+ remote name too.
+ """
+ if not os.environ.get('TESTING', False):
+ cmd = ['git', '-C', conf.srcdir(), 'symbolic-ref', '--short', 'HEAD']
+ out = run_collect_output(cmd).strip()
+ return out
+ return 'master'
+
+ def link_memory_usage(self):
+ """Guesstimate the maximum link memory usage for this build.
+ We are only building master here so we will just use that value
+ """
+ # Determinited experimentally.
+ usages = {'master': 3.5}
+ if self.branch() == 'master':
+ return usages['master']
+ else:
+ raise NotImplementedError(
+ "Unknown link memory usage." + self.branch())
+
+
+# Global storage for configuration object.
+conf = None # type: Configuration
+
+def cmake_builder(target):
+ if not os.getenv("TESTING"):
+ dep.parse_dependencies([here + "/clang_build_dependencies.dep"])
+
+ env = []
+ dyld_path = ""
+ if conf.lto and conf.liblto():
+ dyld_path = conf.liblto()
+ env.extend(["env", "DYLD_LIBRARY_PATH=" + dyld_path])
+
+ cmake_cmd = env + ["/usr/local/bin/cmake", "-G", "Ninja",
+ '-DCMAKE_MAKE_PROGRAM=' + NINJA,
+ "-DCMAKE_INSTALL_PREFIX=" + conf.installdir(),
+ "-DLLVM_ENABLE_PROJECTS=" + conf.llvm_enable_projects,
+ conf.llvmsrcdir()]
+
+ compiler_flags = conf.compiler_flags
+ max_parallel_links = conf.max_parallel_links
+
+ if conf.lto:
+ if conf.thinlto:
+ cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS=1"]
+ else:
+ cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS=" + str(max_link_jobs())]
+ cmake_cmd += ['-DLLVM_BUILD_EXAMPLES=Off']
+ if not max_parallel_links:
+ max_parallel_links = 1
+ if dyld_path:
+ cmake_cmd += ['-DDYLD_LIBRARY_PATH=' + dyld_path]
+ else:
+ cmake_cmd += ['-DLLVM_ENABLE_LTO=Off']
+ cmake_cmd += ['-DLLVM_BUILD_EXAMPLES=On']
+
+ cmake_cmd += ["-DCMAKE_MACOSX_RPATH=On"]
+
+ libtool_path = query_sys_tool("macosx", "libtool")
+ if libtool_path:
+ cmake_cmd += ['-DCMAKE_LIBTOOL=' + libtool_path]
+
+ if compiler_flags:
+ cmake_cmd += ["-DCMAKE_C_FLAGS={}".format(' '.join(compiler_flags)),
+ "-DCMAKE_CXX_FLAGS={}".format(' '.join(compiler_flags))]
+
+ if max_parallel_links is not None:
+ cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS={}".format(max_parallel_links)]
+
+ if conf.CC():
+ cmake_cmd += ['-DCMAKE_C_COMPILER=' + conf.CC(),
+ '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"]
+
+ if conf.cmake_build_type:
+ cmake_cmd += ["-DCMAKE_BUILD_TYPE=" + conf.cmake_build_type]
+ elif conf.debug:
+ cmake_cmd += ["-DCMAKE_BUILD_TYPE=Debug"]
+ else:
+ cmake_cmd += ["-DCMAKE_BUILD_TYPE=Release"]
+
+ cmake_cmd += ["-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On"]
+
+ for flag in conf.cmake_flags:
+ cmake_cmd += [flag]
+
+ if conf.assertions:
+ cmake_cmd += ["-DLLVM_ENABLE_ASSERTIONS=On"]
+ else:
+ cmake_cmd += ["-DLLVM_ENABLE_ASSERTIONS=Off"]
+
+ if conf.globalisel:
+ cmake_cmd += ["-DLLVM_BUILD_GLOBAL_ISEL=ON"]
+
+ lit_flags = ['--xunit-xml-output=testresults.xunit.xml', '-v', '--timeout=600']
+ if conf.max_parallel_tests:
+ lit_flags += ['-j', conf.max_parallel_tests]
+ cmake_cmd += ['-DLLVM_LIT_ARGS={}'.format(' '.join(lit_flags))]
+
+ ninja_cmd = env + ["/usr/local/bin/ninja", '-v']
+ if conf.j_level is not None:
+ ninja_cmd += ["-j", conf.j_level]
+
+ if target == 'all' or target == 'build':
+ header("Cmake")
+ run_cmd(conf.builddir(), cmake_cmd)
+ footer()
+ header("Ninja build")
+
+ # Build all if nothing is passed by the user.
+ passed_target = conf.cmake_build_targets
+ build_target = passed_target if passed_target else ['all']
+ run_cmd(conf.builddir(), ninja_cmd + build_target)
+ footer()
+ if conf.noinstall:
+ header("Skip install")
+ else:
+ header("Ninja install")
+ run_cmd(conf.builddir(), ninja_cmd + ['install'])
+ build_upload_artifact()
+ footer()
+ # Run all the test targets.
+ ninja_cmd.extend(['-k', '0'])
+ if target == 'all' or target == 'test' or target == 'testlong':
+ header("Ninja test")
+
+ targets = [
+ 'check-all'] if target == 'testlong' or target == 'all' else conf.cmake_test_targets
+
+ if not targets:
+ # testlong and all do check all, otherwise check and check-clang
+ # unless the user asked for something else.
+ targets = ['check', 'check-clang']
+
+ run_cmd(conf.builddir(), ninja_cmd + targets)
+ footer()
+
+
+def clang_builder(target):
+ """Build to set of commands to compile and test apple-clang"""
+
+ # get rid of old archives from prior builds
+ run_ws(['sh', '-c', 'rm -rfv *gz'])
+
+ if target == "all" or target == "build":
+ # Clean the build directory.
+ run_ws(['rm', '-rf', 'clang.roots'])
+
+ sdk_name = 'macosx'
+
+ sdkroot = query_sdk_path(sdk_name)
+ libtool_path = query_sys_tool(sdk_name, "libtool")
+
+ clang_br = os.path.join(conf.workspace, conf._build_dir)
+ next_section("Build Directory")
+ run_ws(["mkdir", "-p", clang_br])
+
+ toolchain = '/Applications/Xcode.app/Contents/Developer' \
+ '/Toolchains/XcodeDefault.xctoolchain'
+
+ env = []
+ dyld_path = ""
+ if conf.lto and conf.liblto():
+ dyld_path = conf.liblto()
+ env.extend(["env", "DYLD_LIBRARY_PATH=" + dyld_path])
+
+ next_section("Build Clang")
+ if conf.nobootstrap:
+ if conf.debug or conf.device:
+ assert False, "Invalid parameter for clang-builder."
+ run_cmd(clang_br, ['mkdir',
+ './Build',
+ './Root'])
+ install_prefix = conf.installdir()
+
+ # Infer which CMake cache file to use. If ThinLTO we select a specific one.
+ cmake_cachefile_thinlto = ''
+ if conf.thinlto:
+ cmake_cachefile_thinlto = '-ThinLTO'
+ cmake_cachefile = '{}/clang/cmake/caches/Apple-stage2{}.cmake'.format(
+ conf.srcdir(), cmake_cachefile_thinlto)
+
+ cmake_command = env + ["/usr/local/bin/cmake", '-G', 'Ninja', '-C',
+ cmake_cachefile,
+ '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format(
+ "TRUE" if conf.assertions else "FALSE"),
+ '-DCMAKE_BUILD_TYPE=RelWithDebInfo',
+ '-DLLVM_ENABLE_PROJECTS={}'.format(conf.llvm_enable_projects),
+ '-DCMAKE_MAKE_PROGRAM=' + NINJA,
+ '-DLLVM_VERSION_PATCH=99',
+ '-DLLVM_VERSION_SUFFIX=""',
+ '-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On',
+ '-DCLANG_COMPILER_RT_CMAKE_ARGS={}/compiler-rt/cmake/caches/Apple.cmake'.format(
+ conf.srcdir()),
+ '-DCOMPILER_RT_BUILD_SANITIZERS=On',
+ '-DCMAKE_INSTALL_PREFIX={}'.format(
+ install_prefix),
+ '-DCLANG_REPOSITORY_STRING={}'.format(
+ conf.branch()),
+ '-DCLANG_APPEND_VC_REV=On',
+ '-DLLVM_BUILD_TESTS=On',
+ '-DLLVM_INCLUDE_TESTS=On',
+ '-DCLANG_INCLUDE_TESTS=On',
+ '-DLLVM_INCLUDE_UTILS=On',
+ '-DLIBCXX_INSTALL_HEADERS=On',
+ '-DLIBCXX_OVERRIDE_DARWIN_INSTALL=On',
+ '-DLIBCXX_INSTALL_LIBRARY=Off',
+ '-DCMAKE_MACOSX_RPATH=On',
+ ]
+
+ if dyld_path:
+ cmake_command += ['-DDYLD_LIBRARY_PATH=' + dyld_path]
+
+ if libtool_path:
+ cmake_command += ['-DCMAKE_LIBTOOL=' + libtool_path]
+
+ if conf.CC():
+ cmake_command.extend(['-DCMAKE_C_COMPILER=' + conf.CC(),
+ '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"])
+
+ lit_flags = ['--xunit-xml-output=testresults.xunit.xml', '-v', '--timeout=600']
+ if conf.max_parallel_tests:
+ lit_flags += ['-j', conf.max_parallel_tests]
+ cmake_command.extend(
+ ['-DLLVM_LIT_ARGS={}'.format(' '.join(lit_flags))])
+
+ if conf.thinlto:
+ cmake_command.extend(["-DLLVM_PARALLEL_LINK_JOBS=1"])
+ elif conf.lto:
+ cmake_command.extend(
+ ["-DLLVM_PARALLEL_LINK_JOBS=" + str(max_link_jobs())])
+ else:
+ cmake_command.extend(['-DLLVM_ENABLE_LTO=Off'])
+ cmake_command.extend([
+ '-DCMAKE_C_FLAGS_RELWITHDEBINFO:STRING=-O2 -gline-tables-only -DNDEBUG',
+ '-DCMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -gline-tables-only -DNDEBUG'])
+
+ for flag in conf.cmake_flags:
+ cmake_command += [flag]
+
+ cmake_command.append(conf.llvmsrcdir())
+ run_cmd(os.path.join(clang_br, 'Build'), cmake_command)
+ next_section("Ninja")
+ run_cmd(os.path.join(clang_br, 'Build'), [NINJA, '-v', 'install'])
+
+ build_upload_artifact()
+
+ else:
+ # Two stage build, via the make files.
+ print 'Stage two compile TBD in near future'
+
+ if not conf.device and (target == "test" or target == "all"):
+ # Add steps to run the tests.
+ next_section("Tests")
+ # Auto detect bootstrap and non-bootstrap.
+ obj_dir = os.path.join(conf._build_dir,
+ 'Objects/obj-llvm/tools/clang/stage2-bins/')
+ if not os.path.exists(obj_dir):
+ obj_dir = os.path.join(conf._build_dir, 'Build/')
+ obj_dir = os.path.join(conf.workspace, obj_dir)
+
+ cmd = [NINJA, '-v', '-k', '0', 'check-all']
+
+ if conf.assertions:
+ cmd[-1] += ' --param use_gmalloc=1 ' \
+ '--param gmalloc_path=$(xcodebuild -find-library' \
+ ' libgmalloc.dylib)'
+ run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'})
+
+
+def parse_settings_from_output(working_dir, cmd):
+ old_dir = os.getcwd()
+ try:
+ os.chdir(working_dir)
+ assignment_regex = re.compile(r"^\s+([^\s=]+)\s*=\s*(.+)$")
+ settings = {}
+ for line in subprocess.check_output(cmd).splitlines(True):
+ match = assignment_regex.match(line)
+ if match:
+ settings[match.group(1)] = match.group(2)
+ return settings
+ finally:
+ os.chdir(old_dir)
+
+
+def lldb_builder():
+ """Do an Xcode build of lldb."""
+
+ # Wipe the build folder
+
+ header("Clean LLDB build directory")
+ if os.path.exists(conf.lldbbuilddir()):
+ shutil.rmtree(conf.lldbbuilddir())
+ footer()
+
+ # Build into the build folder
+ build_configuration = "Release"
+
+ xcodebuild_cmd = [
+ "xcodebuild",
+ "-arch", "x86_64",
+ "-configuration", build_configuration,
+ "-scheme", "desktop",
+ "-derivedDataPath", conf.lldbbuilddir()
+ # It is too fragile to use the Xcode debugserver. If we add new
+ # command line arguments to debugserver, the older Xcode debugserver
+ # will fall over and not run. By commenting out this flag, we
+ # are requiring the builder to have the lldb_codesign code signing
+ # certificate and we are ensuring we are testing the latest debugserver
+ # from lldb.
+ # "DEBUGSERVER_USE_FROM_SYSTEM=1"
+ ]
+
+ header("Build Xcode desktop scheme")
+ run_cmd("lldb", xcodebuild_cmd)
+ footer()
+
+ header("Gather Xcode build settings")
+ xcodebuild_cmd.append("-showBuildSettings")
+ settings = parse_settings_from_output("lldb", xcodebuild_cmd)
+ footer()
+
+ build_dir = settings.get("BUILD_DIR", None)
+ built_products_dir = settings.get("BUILT_PRODUCTS_DIR", None)
+ if build_dir is None or built_products_dir is None:
+ raise Exception("failed to retrieve build-related directories "
+ "from Xcode")
+
+ llvm_build_dir = settings.get("LLVM_BUILD_DIR", None)
+ llvm_build_dir_arch = settings.get("LLVM_BUILD_DIR_ARCH", None)
+ if llvm_build_dir is None or llvm_build_dir_arch is None:
+ raise Exception("failed to retrieve LLVM build-related settings "
+ "from Xcode")
+ llvm_build_bin_dir = os.path.join(llvm_build_dir, llvm_build_dir_arch, "bin")
+ built_clang_path = os.path.join(llvm_build_bin_dir, "clang")
+ built_filecheck_path = os.path.join(llvm_build_bin_dir, "FileCheck")
+ effective_clang = os.environ.get("LLDB_PYTHON_TESTSUITE_CC",
+ built_clang_path)
+
+ # Run C++ test suite (gtests)
+
+ xcodebuild_cmd = [
+ "xcodebuild",
+ "-arch", "x86_64",
+ "-configuration", build_configuration,
+ "-scheme", "lldb-gtest",
+ "-derivedDataPath", conf.lldbbuilddir(),
+ # See notes above.
+ # "DEBUGSERVER_USE_FROM_SYSTEM=1"
+ ]
+
+ header("Build Xcode lldb-gtest scheme")
+ run_cmd("lldb", xcodebuild_cmd)
+ footer()
+
+ # Run LLDB Python test suite for archs defined in LLDB_TEST_ARCHS
+ for arch in conf.lldb_test_archs:
+ results_file = os.path.join(build_dir,
+ "test-results-{}.xml".format(arch))
+ python_testsuite_cmd = [
+ "/usr/bin/python",
+ "test/dotest.py",
+ "--executable", os.path.join(built_products_dir, "lldb"),
+ "-C", effective_clang,
+ "--arch", arch,
+ "--results-formatter",
+ "lldbsuite.test_event.formatter.xunit.XunitFormatter",
+ "--results-file", results_file,
+ "--rerun-all-issues",
+ "--env", "TERM=vt100",
+ "-O--xpass=ignore",
+ "--dsymutil="+os.path.join(os.path.dirname(effective_clang), 'dsymutil'),
+ "--filecheck="+built_filecheck_path
+ ]
+
+ header("Run LLDB Python-based test suite ({} targets)".format(arch))
+ # For the unit tests, we don't want to stop the build if there are
+ # build errors. We allow the JUnit/xUnit parser to pick this up.
+ print repr(python_testsuite_cmd)
+ run_cmd_errors_okay("lldb", python_testsuite_cmd)
+ footer()
+
+
+def lldb_cmake_builder():
+ """Do a CMake build of lldb."""
+
+ test_dir = os.path.join(conf.workspace, 'test')
+ log_dir = os.path.join(test_dir, 'logs')
+ results_file = os.path.join(test_dir, 'results.xml')
+ dest_dir = os.path.join(conf.workspace, 'results', 'lldb')
+ run_ws(["mkdir", "-p", conf.lldbbuilddir()])
+ cmake_build_type = conf.cmake_build_type if conf.cmake_build_type else 'RelWithDebInfo'
+ header("Configure")
+ dotest_args=['--arch', 'x86_64', '--build-dir',
+ conf.lldbbuilddir()+'/lldb-test-build.noindex',
+ '-s='+log_dir,
+ '-t',
+ '--env', 'TERM=vt100']
+ dotest_args.extend(conf.dotest_flags)
+ cmake_cmd = ["/usr/local/bin/cmake", '-G', 'Ninja',
+ conf.srcdir(),
+ '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format(
+ "TRUE" if conf.assertions else "FALSE"),
+ '-DLLVM_ENABLE_PROJECTS='+conf.llvm_enable_projects,
+ '-DCMAKE_BUILD_TYPE='+cmake_build_type,
+ '-DCMAKE_MAKE_PROGRAM=' + NINJA,
+ '-DLLVM_VERSION_PATCH=99',
+ '-DLLVM_ENABLE_MODULES=On',
+ '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
+ '-DCMAKE_INSTALL_PREFIX="%s"'%dest_dir,
+ '-DLLDB_TEST_USER_ARGS='+';'.join(dotest_args),
+ '-DLLVM_LIT_ARGS=--xunit-xml-output=%s -v'%results_file]
+ cmake_cmd.extend(conf.cmake_flags)
+
+ if conf.CC():
+ cmake_cmd.extend(['-DCMAKE_C_COMPILER=' + conf.CC(),
+ '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"])
+
+ run_cmd(conf.lldbbuilddir(), cmake_cmd)
+ footer()
+
+ header("Build")
+ run_cmd(conf.lldbbuilddir(), [NINJA, '-v'])
+ footer()
+
+ header("Run Tests")
+ run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo'])
+ run_cmd(conf.lldbbuilddir(), ['/usr/bin/env', 'TERM=vt100', NINJA, '-v',
+ 'check-lldb'])
+ footer()
+
+
+def static_analyzer_benchmarks_builder():
+ """Run static analyzer benchmarks"""
+ header("Static Analyzer Benchmarks")
+
+ benchmark_script = conf.workspace + "/utils-analyzer/SATestBuild.py"
+ benchmarks_dir = conf.workspace + "/test-suite-ClangAnalyzer/"
+
+ compiler_bin_dir = conf.workspace + "/host-compiler/bin/"
+ scanbuild_bin_dir = conf.workspace + "/tools-scan-build/bin/"
+
+ old_path = os.environ.get("PATH", "")
+ env = dict(os.environ, PATH=compiler_bin_dir + os.pathsep +
+ scanbuild_bin_dir + os.pathsep +
+ old_path)
+
+ benchmark_cmd = [benchmark_script,
+ "--strictness", "0"
+ ]
+ run_cmd(benchmarks_dir, benchmark_cmd, env=env)
+
+ footer()
+
+
+def http_download(url, dest):
+ """Safely download url to dest.
+
+ Print error and exit if download fails.
+ """
+ try:
+ print "GETting", url, "to", dest, "...",
+ f = urlopen(url)
+ # Open our local file for writing
+ with open(dest, "wb") as local_file:
+ local_file.write(f.read())
+
+ except HTTPError, e:
+ print
+ print "HTTP Error:", e.code, url
+ sys.exit(1)
+
+ except URLError, e:
+ print
+ print "URL Error:", e.reason, url
+ sys.exit(1)
+ print "done."
+
+
+def create_builddirs():
+ paths = [conf.builddir(), conf.installdir()]
+ for p in paths:
+ if not os.path.exists(p):
+ os.makedirs(p)
+
+
+def fetch_compiler():
+ local_name = "host-compiler.tar.gz"
+ url = conf.host_compiler_url + "/" + conf.artifact_url
+ header("Fetching Compiler")
+ http_download(url, conf.workspace + "/" + local_name)
+ print "Decompressing..."
+ if os.path.exists(conf.workspace + "/host-compiler"):
+ shutil.rmtree(conf.workspace + "/host-compiler")
+ os.mkdir(conf.workspace + "/host-compiler")
+ run_cmd(conf.workspace + "/host-compiler/",
+ ['tar', 'zxf', "../" + local_name])
+ os.unlink(local_name)
+ footer()
+
+def build_upload_artifact():
+ """Create artifact for this build, and upload to server."""
+ if conf.noupload:
+ print 'Not uploading artificats'
+ return
+ header("Uploading Artifact")
+ prop_file = "last_good_build.properties"
+
+ # TODO : azhar : Update artifact name
+ artifact_name = "clang-t{}-b{}.tar.gz".format(conf.build_id,
+ conf.build_number)
+ new_url = conf.job_name + "/" + artifact_name
+
+ with open(prop_file, 'w') as prop_fd:
+ prop_fd.write("GIT_SHA={}\n".format(conf.git_sha))
+ prop_fd.write("ARTIFACT={}\n".format(new_url))
+
+ # The .a's are big and we don't need them later. Drop the LLVM and clang
+ # libraries, but keep the libraries from compiler-rt.
+ tar = ["tar", "zcvf", "../" + artifact_name, "--exclude=*libLLVM*.a",
+ "--exclude=*libclang[A-Z]*.a", "."]
+
+ run_cmd(conf.installdir(), tar)
+
+ mkdir_cmd = ["ssh", "buildslave@" + SERVER, "mkdir", "-p", "/Library/WebServer/Documents/artifacts/" + conf.job_name]
+
+ run_cmd(conf.workspace, mkdir_cmd)
+
+ upload_cmd = ["scp", artifact_name,
+ "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" +
+ conf.job_name + "/"]
+
+ run_cmd(conf.workspace, upload_cmd)
+
+ upload_cmd = ["scp", prop_file,
+ "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" +
+ conf.job_name + "/"]
+
+ run_cmd(conf.workspace, upload_cmd)
+
+ ln_cmd = ["ssh", "buildslave@" + SERVER,
+ "ln", "-fs", "/Library/WebServer/Documents/artifacts/" +
+ conf.job_name + "/" + artifact_name,
+ "/Library/WebServer/Documents/artifacts/" +
+ conf.job_name + "/latest"]
+
+ run_cmd(conf.workspace, ln_cmd)
+
+ lng_cmd = ["ssh", "buildslave@" + SERVER,
+ "ln", "-fs", "/Library/WebServer/Documents/artifacts/" +
+ conf.job_name + "/" + artifact_name,
+ "/Library/WebServer/Documents/artifacts/" +
+ conf.job_name + "/g" + conf.git_sha]
+ run_cmd(conf.workspace, lng_cmd)
+
+
+def run_cmd(working_dir, cmd, env=None, sudo=False, err_okay=False):
+ """Run a command in a working directory, and make sure it returns zero."""
+ assert type(cmd) == list, "Not a list: {}".format(type(cmd))
+ old_cwd = os.getcwd()
+ if env:
+ envs = []
+ for key, value in env.items():
+ envs.append("{}={}".format(key, value))
+ cmd = ["env"] + envs + cmd
+ if sudo:
+ cmd = ['sudo'] + cmd
+
+ cmd_to_print = ' '.join([quote_sh_string(x) for x in cmd])
+ sys.stdout.write("cd {}\n{}\n".format(working_dir, cmd_to_print))
+ sys.stdout.flush()
+ return_code = 0
+ start_time = datetime.datetime.now()
+ if not os.environ.get('TESTING', False):
+ try:
+ os.chdir(working_dir)
+ subprocess.check_call(cmd)
+ os.chdir(old_cwd)
+ except subprocess.CalledProcessError as excpt:
+ if not err_okay:
+ raise excpt
+ else:
+ logging.info("Ignoring failed command.")
+ return_code = excpt.returncode
+ end_time = datetime.datetime.now()
+
+ logging.info("Command took {} seconds".format(
+ (end_time - start_time).seconds))
+ return return_code
+
+
+def run_cmd_errors_okay(working_dir, cmd, env=None):
+ """Run a command in a working directory, reporting return value.
+ Non-zero exit codes do not generate an exception.
+ """
+ old_cwd = os.getcwd()
+ cmd_to_print = ' '.join([quote_sh_string(x) for x in cmd])
+ sys.stdout.write("cd {}\n{}\n".format(working_dir, cmd_to_print))
+ sys.stdout.flush()
+
+ start_time = datetime.datetime.now()
+ if not os.environ.get('TESTING', False):
+ try:
+ os.chdir(working_dir)
+ result = subprocess.call(cmd, env=env)
+ finally:
+ os.chdir(old_cwd)
+ end_time = datetime.datetime.now()
+
+ logging.info("Command took {} seconds: return code {}".format(
+ (end_time - start_time).seconds, result))
+
+
+KNOWN_TARGETS = ['all', 'build', 'test', 'testlong']
+KNOWN_BUILDS = ['clang', 'cmake', 'lldb', 'lldb-cmake', 'fetch', 'artifact',
+ 'static-analyzer-benchmarks']
+
+
+def query_sdk_path(sdk_name):
+ """Get the path to the sdk named using xcrun.
+
+ When $TESTING define, just give a dummy back. We do this because xcrun
+ could fail if the sdk you want is not installed, and that is silly for
+ testing.
+ """
+
+ if not os.environ.get('TESTING', False):
+ cmd = ['xcrun', '--sdk', sdk_name, '--show-sdk-path']
+ return run_collect_output(cmd).strip()
+ else:
+ return "/Applications/Xcode.app/Contents/Developer/Platforms/" \
+ "MacOSX.platform/Developer/SDKs/MacOSX10.10.sdk"
+
+
+def max_link_jobs():
+ """Link jobs take about 3.6GB of memory, max."""
+ mem_str = run_collect_output(["sysctl", "hw.memsize"])
+ mem = float(mem_str.split()[1].strip())
+ mem = mem / (1024.0 ** 3) # Conver to GBs.
+ return int(math.ceil(mem / conf.link_memory_usage()))
+
+
+TEST_VALS = {"sysctl hw.ncpu": "hw.ncpu: 8\n",
+ "sysctl hw.memsize": "hw.memsize: 8589934592\n",
+ "xcrun --sdk iphoneos --show-sdk-path": "/Foo/bar",
+ "/usr/bin/xcrun svn upgrade": "",
+ }
+
+
+ at contextmanager
+def cwd(path):
+ last_cwd = os.getcwd()
+ if path:
+ os.chdir(path)
+ try:
+ yield
+ finally:
+ os.chdir(last_cwd)
+
+
+def run_collect_output(cmd, working_dir=None, stderr=None):
+ """Run cmd and then return the output.
+
+ If working_dir is supplied the cmd will run in
+ with a context manager in working_dir.
+ """
+ if os.getenv("TESTING"):
+ print 'TV: ' + ' '.join(cmd)
+ return TEST_VALS[' '.join(cmd)]
+
+ with cwd(working_dir):
+ return subprocess.check_output(cmd, stderr=stderr)
+
+
+def query_sys_tool(sdk_name, tool_name):
+ """Get the path of system tool
+
+ When $TESTING define, just give a dummy back.
+ """
+
+ if not os.environ.get('TESTING', False):
+ cmd = ['xcrun', '--sdk', sdk_name, '--find', tool_name]
+ return run_collect_output(cmd).strip()
+ else:
+ return "/usr/bin/" + tool_name
+
+
+def run_ws(cmd, env=None):
+ """Wrapper to call run_cmd in local workspace.
+
+ Since 99 percent of the time, that is where you want to call things from.
+ """
+ return run_cmd(conf.workspace, cmd, env)
+
+
+def parse_args():
+ """Get the command line arguments, and make sure they are correct."""
+
+ parser = argparse.ArgumentParser(
+ description='Build and test compilers and other things.')
+
+ parser.add_argument("build_type",
+ help="The kind of build to trigger.",
+ choices=KNOWN_BUILDS)
+
+ parser.add_argument("build_target",
+ nargs='?',
+ help="The targets to call (build, check, etc).",
+ choices=KNOWN_TARGETS)
+
+ parser.add_argument('--assertions', dest='assertions', action='store_true')
+ parser.add_argument('--lto', dest='lto', action='store_true')
+ parser.add_argument('--thinlto', dest='thinlto', action='store_true')
+ parser.add_argument('--debug', dest='debug', action='store_true')
+ parser.add_argument('--cmake-type', dest='cmake_build_type',
+ help="Override cmake type Release, Debug, "
+ "RelWithDebInfo and MinSizeRel")
+ parser.add_argument('--cmake-flag', dest='cmake_flags',
+ action='append', default=[],
+ help='Set an arbitrary cmake flag')
+ parser.add_argument('--dotest-flag', dest='dotest_flags',
+ action='append', default=[],
+ help='Set an arbitrary lldb dotest.py flag')
+ parser.add_argument('--cmake-test-target', dest='cmake_test_targets',
+ action='append', default=[],
+ help='Targets to build during testing')
+ parser.add_argument('--cmake-build-target', dest='cmake_build_targets',
+ action='append', default=[],
+ help='Targets to build during building.')
+ parser.add_argument('--compiler-flag', dest='compiler_flags',
+ action='append', default=[],
+ help='Set an arbitrary compiler flag')
+ parser.add_argument('--noupload', dest='noupload', action='store_true')
+ parser.add_argument('--noinstall', dest='noinstall', action='store_true',
+ help="Disable the install stage, build only.")
+ parser.add_argument('--globalisel', dest='globalisel',
+ action='store_true', help="Turn on the experimental"
+ " GlobalISel CMake flag.")
+ parser.add_argument('--projects', dest='llvm_enable_projects',
+ default="clang;clang-tools-extra;compiler-rt;libcxx",
+ help="Semicolon seperated list of projects to build.")
+
+ args = parser.parse_args()
+ if args.thinlto:
+ args.lto = True
+ return args
+
+
+def main():
+ """Run a build based on command line args and ENV."""
+ global conf
+ args = parse_args()
+ conf = Configuration(args)
+
+ create_builddirs()
+ try:
+ if args.build_type == 'clang':
+ clang_builder(args.build_target)
+ elif args.build_type == 'lldb':
+ lldb_builder()
+ elif args.build_type == 'lldb-cmake':
+ lldb_cmake_builder()
+ elif args.build_type == 'cmake':
+ cmake_builder(args.build_target)
+ elif args.build_type == 'fetch':
+ fetch_compiler()
+ elif args.build_type == 'artifact':
+ build_upload_artifact()
+ elif args.build_type == 'static-analyzer-benchmarks':
+ static_analyzer_benchmarks_builder()
+ except subprocess.CalledProcessError as exct:
+ print "Command failed", exct.message
+ print "Command:", exct.cmd
+ sys.exit(1)
+
+
+if __name__ == '__main__':
+ main()
More information about the llvm-commits
mailing list