[zorg] r317539 - Open source task tool + tasks
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 6 19:18:32 PST 2017
Author: matze
Date: Mon Nov 6 19:18:31 2017
New Revision: 317539
URL: http://llvm.org/viewvc/llvm-project?rev=317539&view=rev
Log:
Open source task tool + tasks
This open sources the 'task' tool which we created and used for internal
jobs for a while. It is basically a thin layer that allows you to put
self-contained task descriptions into a git repository. Those can then
be used as part of CI jobs, but also allow you to run tasks locally
without jenkins installed.
Also add a bunch of tasks and cmake caches to perfrom some lnt-ctmask,
lnt-test-suite and compiler-verifiers builds.
This will initially be used by the greendragon jobs.
Added:
zorg/trunk/tasks/
zorg/trunk/tasks/.gitignore
zorg/trunk/tasks/README.md
zorg/trunk/tasks/bin/
zorg/trunk/tasks/bin/build (with props)
zorg/trunk/tasks/bin/cmake (with props)
zorg/trunk/tasks/bin/ninja (with props)
zorg/trunk/tasks/cmake/
zorg/trunk/tasks/cmake/caches/
zorg/trunk/tasks/cmake/caches/global-isel.cmake
zorg/trunk/tasks/cmake/caches/opt-O0-g.cmake
zorg/trunk/tasks/cmake/caches/opt-O3-flto.cmake
zorg/trunk/tasks/cmake/caches/opt-O3.cmake
zorg/trunk/tasks/cmake/caches/opt-Os-flto.cmake
zorg/trunk/tasks/cmake/caches/opt-Os.cmake
zorg/trunk/tasks/cmake/caches/opt-Oz.cmake
zorg/trunk/tasks/cmake/caches/target-arm64-iphoneos.cmake
zorg/trunk/tasks/cmake/caches/target-x86_64-macosx.cmake
zorg/trunk/tasks/cmake/caches/target-x86_64h-macosx.cmake
zorg/trunk/tasks/cmake/caches/util/
zorg/trunk/tasks/cmake/caches/util/arch_flags_toolchain.cmake
zorg/trunk/tasks/cmake/caches/util/xcode_sdk.cmake
zorg/trunk/tasks/cmake/caches/verify-machineinstrs.cmake
zorg/trunk/tasks/hello_world.c
zorg/trunk/tasks/hello_world3.sh
zorg/trunk/tasks/lnt-ctmark.sh
zorg/trunk/tasks/lnt-test-suite.sh
zorg/trunk/tasks/repos/
zorg/trunk/tasks/repos/clang-stage1-configure-RA.json
zorg/trunk/tasks/repos/compiler.json (with props)
zorg/trunk/tasks/repos/llvm.json
zorg/trunk/tasks/repos/lnt.json
zorg/trunk/tasks/repos/test-suite.json
zorg/trunk/tasks/task (with props)
zorg/trunk/tasks/tasktool/
zorg/trunk/tasks/tasktool/README.md
zorg/trunk/tasks/tasktool/bin/
zorg/trunk/tasks/tasktool/bin/build (with props)
zorg/trunk/tasks/tasktool/bin/task (with props)
zorg/trunk/tasks/tasktool/setup.py
zorg/trunk/tasks/tasktool/tasktool/
zorg/trunk/tasks/tasktool/tasktool/__init__.py
zorg/trunk/tasks/tasktool/tasktool/build.py
zorg/trunk/tasks/tasktool/tasktool/hooks/
zorg/trunk/tasks/tasktool/tasktool/hooks/check-clean-repo.sh
zorg/trunk/tasks/tasktool/tasktool/hooks/check-config.sh
zorg/trunk/tasks/tasktool/tasktool/hooks/exec-try-build (with props)
zorg/trunk/tasks/tasktool/tasktool/hooks/footer_sshrun
zorg/trunk/tasks/tasktool/tasktool/hooks/header
zorg/trunk/tasks/tasktool/tasktool/hooks/header_jenkinsrun
zorg/trunk/tasks/tasktool/tasktool/hooks/header_sshrun
zorg/trunk/tasks/tasktool/tasktool/hooks/header_submit
zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-common.sh
zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-id.sh
zorg/trunk/tasks/tasktool/tasktool/hooks/mk-jenkinsrun-build (with props)
zorg/trunk/tasks/tasktool/tasktool/hooks/mk-sshrun-build (with props)
zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-build (with props)
zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-results (with props)
zorg/trunk/tasks/tasktool/tasktool/hooks/mk-try-build (with props)
zorg/trunk/tasks/tasktool/tasktool/hooks/sshrun (with props)
zorg/trunk/tasks/tasktool/tasktool/hooks/submit (with props)
zorg/trunk/tasks/tasktool/tasktool/hooks/try.sb
zorg/trunk/tasks/tasktool/tasktool/repos/
zorg/trunk/tasks/tasktool/tasktool/repos/__init__.py
zorg/trunk/tasks/tasktool/tasktool/repos/artifact_server.py
zorg/trunk/tasks/tasktool/tasktool/repos/existing.py
zorg/trunk/tasks/tasktool/tasktool/repos/git.py
zorg/trunk/tasks/tasktool/tasktool/repos/url.py
zorg/trunk/tasks/tasktool/tasktool/task.py
zorg/trunk/tasks/tasktool/tasktool/utils.py
zorg/trunk/tasks/test-suite-verify-machineinstrs.sh
zorg/trunk/tasks/utils/
zorg/trunk/tasks/utils/lnt_check_no_errors.sh
zorg/trunk/tasks/utils/lnt_move_results.sh
zorg/trunk/tasks/utils/lnt_submit.sh
zorg/trunk/tasks/utils/lnt_test_suite.sh
zorg/trunk/tasks/utils/normalize_compiler.sh
zorg/trunk/tasks/utils/pip_install.sh
zorg/trunk/tasks/utils/test_suite_compile.sh
zorg/trunk/tasks/utils/venv.sh
zorg/trunk/tasks/utils/venv_lit.sh
zorg/trunk/tasks/utils/venv_lnt.sh
Added: zorg/trunk/tasks/.gitignore
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/.gitignore?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/.gitignore (added)
+++ zorg/trunk/tasks/.gitignore Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+/repos.try
+/config
+/results
+*.pyc
Added: zorg/trunk/tasks/README.md
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/README.md?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/README.md (added)
+++ zorg/trunk/tasks/README.md Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+This directory contains task definitions for various greendragon jenkins jobs.
+
+See `tasktool/README.md` for how these work.
Added: zorg/trunk/tasks/bin/build
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/bin/build?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/bin/build (added)
+++ zorg/trunk/tasks/bin/build Mon Nov 6 19:18:31 2017
@@ -0,0 +1 @@
+link ../tasktool/bin/build
\ No newline at end of file
Propchange: zorg/trunk/tasks/bin/build
------------------------------------------------------------------------------
svn:special = *
Added: zorg/trunk/tasks/bin/cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/bin/cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/bin/cmake (added)
+++ zorg/trunk/tasks/bin/cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+#!/bin/sh
+# We expect cmake to be installed in /usr/local/bin/cmake
+/usr/local/bin/cmake "$@"
Propchange: zorg/trunk/tasks/bin/cmake
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/bin/ninja
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/bin/ninja?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/bin/ninja (added)
+++ zorg/trunk/tasks/bin/ninja Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+#!/bin/sh
+# We expect cmake to be installed in /usr/local/bin/cmake
+/usr/local/bin/ninja "$@"
Propchange: zorg/trunk/tasks/bin/ninja
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/cmake/caches/global-isel.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/global-isel.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/global-isel.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/global-isel.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1 @@
+set(OPTFLAGS "${OPTFLAGS} -mllvm -global-isel -mllvm -global-isel-abort=2" CACHE STRING "")
Added: zorg/trunk/tasks/cmake/caches/opt-O0-g.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/opt-O0-g.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/opt-O0-g.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/opt-O0-g.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+set(OPTFLAGS "${OPTFLAGS} -O0 -g")
+set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "")
+set(CMAKE_C_FLAGS_DEBUG "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG "${OPTFLAGS}" CACHE STRING "")
Added: zorg/trunk/tasks/cmake/caches/opt-O3-flto.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/opt-O3-flto.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/opt-O3-flto.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/opt-O3-flto.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+set(OPTFLAGS "${OPTFLAGS} -O3 -flto")
+set(CMAKE_C_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")
Added: zorg/trunk/tasks/cmake/caches/opt-O3.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/opt-O3.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/opt-O3.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/opt-O3.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+set(OPTFLAGS "${OPTFLAGS} -O3")
+set(CMAKE_C_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")
Added: zorg/trunk/tasks/cmake/caches/opt-Os-flto.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/opt-Os-flto.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/opt-Os-flto.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/opt-Os-flto.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+set(OPTFLAGS "${OPTFLAGS} -Os -flto")
+set(CMAKE_C_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")
Added: zorg/trunk/tasks/cmake/caches/opt-Os.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/opt-Os.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/opt-Os.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/opt-Os.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+set(OPTFLAGS "${OPTFLAGS} -Os")
+set(CMAKE_C_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")
Added: zorg/trunk/tasks/cmake/caches/opt-Oz.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/opt-Oz.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/opt-Oz.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/opt-Oz.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+set(OPTFLAGS "${OPTFLAGS} -Oz")
+set(CMAKE_C_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELEASE "${OPTFLAGS}" CACHE STRING "")
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")
Added: zorg/trunk/tasks/cmake/caches/target-arm64-iphoneos.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/target-arm64-iphoneos.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/target-arm64-iphoneos.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/target-arm64-iphoneos.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+set(XCRUN_FLAGS --toolchain iOS --sdk iphoneos)
+set(CMAKE_OSX_ARCHITECTURES arm64 CACHE STRING "")
+include(${CMAKE_CURRENT_LIST_DIR}/util/xcode_sdk.cmake)
Added: zorg/trunk/tasks/cmake/caches/target-x86_64-macosx.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/target-x86_64-macosx.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/target-x86_64-macosx.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/target-x86_64-macosx.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+set(XCRUN_FLAGS --sdk macosx)
+set(CMAKE_OSX_ARCHITECTURES x86_64 CACHE STRING "")
+include(${CMAKE_CURRENT_LIST_DIR}/util/xcode_sdk.cmake)
Added: zorg/trunk/tasks/cmake/caches/target-x86_64h-macosx.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/target-x86_64h-macosx.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/target-x86_64h-macosx.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/target-x86_64h-macosx.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+set(XCRUN_FLAGS --sdk macosx)
+set(CMAKE_OSX_ARCHITECTURES x86_64h CACHE STRING "")
+include(${CMAKE_CURRENT_LIST_DIR}/util/xcode_sdk.cmake)
Added: zorg/trunk/tasks/cmake/caches/util/arch_flags_toolchain.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/util/arch_flags_toolchain.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/util/arch_flags_toolchain.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/util/arch_flags_toolchain.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,5 @@
+# Hack to get target flags into try_compile. See xcode_sdk.cmake for details.
+if(NOT TEST_SUITE_ARCH_FLAGS AND TEST_SUITE_ARCH_FLAGS_FORWARD)
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${TEST_SUITE_ARCH_FLAGS_FORWARD}" CACHE STRING "" FORCE)
+ set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${TEST_SUITE_ARCH_FLAGS_FORWARD}" CACHE STRING "" FORCE)
+endif()
Added: zorg/trunk/tasks/cmake/caches/util/xcode_sdk.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/util/xcode_sdk.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/util/xcode_sdk.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/util/xcode_sdk.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1,76 @@
+macro(xcrun_find VARIABLE NAME)
+ execute_process(COMMAND xcrun ${XCRUN_FLAGS} -f ${NAME}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ OUTPUT_VARIABLE ${VARIABLE})
+endmacro()
+macro(xcrun_find_update_cache VARIABLE NAME)
+ xcrun_find(${VARIABLE} ${NAME})
+ set(${VARIABLE} "${${VARIABLE}}" CACHE STRING "")
+endmacro()
+
+execute_process(COMMAND xcrun ${XCRUN_FLAGS} --show-sdk-path
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ OUTPUT_VARIABLE SDK_PATH)
+xcrun_find_update_cache(CMAKE_LINKER ld)
+
+if (CMAKE_C_COMPILER)
+ # The user manually specified a compiler which is usually different than
+ # the one coming with the SDK.
+
+ # Construct shim for ranlib, ar, etc.
+ # This is necessary because the ranlib coming with the xcode sdk looks for
+ # ../lib/libLTO.dylib and will therefor pick up libLTO.dylib coming with the
+ # SDK. However for correct behavior it has to pick up libLTO.dylib coming
+ # the clang compiler used.
+ xcrun_find_update_cache(SDK_RANLIB ranlib)
+ get_filename_component(COMPILER_DIR ${CMAKE_C_COMPILER} DIRECTORY)
+
+ macro(create_shim VARIABLE TOOLNAME)
+ xcrun_find(SDK_TOOL_BIN ${TOOLNAME})
+ file(WRITE ${CMAKE_BINARY_DIR}/${TOOLNAME} "
+# Shim to have the tool use the correct libLTO.dylib
+DYLD_LIBRARY_PATH=\"${COMPILER_DIR}/../lib:$DYLD_LIBRARY_PATH\" ${SDK_TOOL_BIN} \"$@\"
+ ")
+ execute_process(COMMAND chmod +x ${CMAKE_BINARY_DIR}/${TOOLNAME})
+ set(${VARIABLE} ${CMAKE_BINARY_DIR}/${TOOLNAME} CACHE STRING "")
+ endmacro()
+
+ create_shim(CMAKE_AR ar)
+ create_shim(CMAKE_NM nm)
+ create_shim(CMAKE_RANLIB ranlib)
+ create_shim(CMAKE_STRIP strip)
+
+ # Skip linking in cmake compiler tests, as the cmake won't pass the -B flags
+ # required to pick up the correct linker to the early compiler tests. However
+ # it does have an option to not link in the early tests.
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY CACHE STRING "")
+else()
+ # Search and use compiler coming with the SDK.
+ # Note that we do not search CMAKE_CXX_COMPILER here. cmake will try
+ # `${CMAKE_C_COMPILER}++` and use that if available.
+ xcrun_find_update_cache(CMAKE_AR ar)
+ xcrun_find_update_cache(CMAKE_NM nm)
+ xcrun_find_update_cache(CMAKE_C_COMPILER clang)
+ xcrun_find_update_cache(CMAKE_RANLIB ranlib)
+ xcrun_find_update_cache(CMAKE_STRIP strip)
+endif()
+
+set(CMAKE_OSX_SYSROOT "${SDK_PATH}" CACHE STRING "")
+
+# Append -B so clang picks up the linker coming with the SDK instead of the
+# one in $PATH.
+xcrun_find(LINKER_PATH ld)
+get_filename_component(LINKER_DIR ${LINKER_PATH} DIRECTORY)
+set(TEST_SUITE_ARCH_FLAGS "${TEST_SUITE_ARCH_FLAGS} -B ${LINKER_DIR}" CACHE STRING "")
+
+# The problem with TEST_SUITE_ARCH_FLAGS is that they will not be used when
+# cmake is testing for features with try_compile().
+#
+# This is a work around for this: try_compile() does honor toolchain files which
+# can chane CMAKE_C_FLAGS.
+#
+# Note that CMAKE_TRY_COMPILE_PLATFORM_VARIABLES used by this is only available
+# in cmake >=3.6
+set(CMAKE_TOOLCHAIN_FILE ${CMAKE_CURRENT_LIST_DIR}/arch_flags_toolchain.cmake CACHE STRING "")
+set(CMAKE_TRY_COMPILE_PLATFORM_VARIABLES "TEST_SUITE_ARCH_FLAGS_FORWARD" CACHE STRING "")
+set(TEST_SUITE_ARCH_FLAGS_FORWARD "${TEST_SUITE_ARCH_FLAGS}" CACHE STRING "")
Added: zorg/trunk/tasks/cmake/caches/verify-machineinstrs.cmake
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/cmake/caches/verify-machineinstrs.cmake?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/cmake/caches/verify-machineinstrs.cmake (added)
+++ zorg/trunk/tasks/cmake/caches/verify-machineinstrs.cmake Mon Nov 6 19:18:31 2017
@@ -0,0 +1 @@
+set(OPTFLAGS "${OPTFLAGS} -Wno-unused-command-line-argument -mllvm -verify-machineinstrs" CACHE STRING "")
Added: zorg/trunk/tasks/hello_world.c
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/hello_world.c?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/hello_world.c (added)
+++ zorg/trunk/tasks/hello_world.c Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+int main(void) {
+ return 2 + 2;
+}
Added: zorg/trunk/tasks/hello_world3.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/hello_world3.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/hello_world3.sh (added)
+++ zorg/trunk/tasks/hello_world3.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,22 @@
+# Tasks can also produce result files. The task runner will check if
+# `${WORKSPACE}/result` exists after the task is finished.
+#
+# The contents will be packed into an archive. Depending on the mode:
+# - `task try` and `task sshrun` will place the archive into the results folder
+# of the llvm-ci-tasks checkout
+# - `task submit` will upload the archive to the A2 server
+# - `task jenkinsrun` does nothing; jenkins is responsible for archiving
+# the results folder.
+
+# Get the compiler as usual.
+build get compiler --from=clang-stag1-configure-RA
+. ${TASKDIR}/utils/normalize_compiler.sh
+
+# Compile an example program.
+compiler/bin/clang ${TASKDIR}/hello_world.c -o hello
+
+# Place the resuling executable into the results folder.
+# You should get an archive containing the "hello" executable when the task is
+# finished.
+mkdir result
+mv hello result
Added: zorg/trunk/tasks/lnt-ctmark.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/lnt-ctmark.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/lnt-ctmark.sh (added)
+++ zorg/trunk/tasks/lnt-ctmark.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,29 @@
+SUBMIT_URL="$(build arg --optional SUBMIT_URL)"
+SUBMIT_NAME="$(build arg --optional SUBMIT_NAME)"
+SUBMIT_ORDER="$(build arg --optional SUBMIT_ORDER)"
+# This is should be used to supply the cmake cache file to use
+LNT_FLAGS="$(build arg --optional LNT_FLAGS)"
+
+# Set defaults for missing/empty parameters.
+: ${SUBMIT_URL:='http://104.154.54.203/db_default/v4/nts/submitRun'}
+: ${SUBMIT_NAME:="${JOB_NAME-}"}
+
+# A generic ctmark run designed to run as a recurring jenkins job with varying
+# cmake caches and submission to an lnt server.
+build get compiler
+build get test-suite
+build get lnt
+
+. "${TASKDIR}"/utils/normalize_compiler.sh
+. "${TASKDIR}"/utils/venv.sh
+. "${TASKDIR}"/utils/venv_lit.sh
+. "${TASKDIR}"/utils/venv_lnt.sh
+
+LNT_FLAGS+=" --cmake-define TEST_SUITE_RUN_BENCHMARKS=Off"
+LNT_FLAGS+=" --build-threads 1"
+LNT_FLAGS+=" --cmake-define TEST_SUITE_SUBDIRS=\"CTMark\""
+. "${TASKDIR}"/utils/lnt_test_suite.sh
+. "${TASKDIR}"/utils/lnt_submit.sh
+
+. "${TASKDIR}"/utils/lnt_move_results.sh
+. "${TASKDIR}"/utils/lnt_check_no_errors.sh
Added: zorg/trunk/tasks/lnt-test-suite.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/lnt-test-suite.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/lnt-test-suite.sh (added)
+++ zorg/trunk/tasks/lnt-test-suite.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,23 @@
+SUBMIT_URL="$(build arg --optional SUBMIT_URL)"
+SUBMIT_NAME="$(build arg --optional SUBMIT_NAME)"
+SUBMIT_ORDER="$(build arg --optional SUBMIT_ORDER)"
+LNT_FLAGS="$(build arg --optional LNT_FLAGS)"
+
+# Set defaults for missing/empty parameters.
+: ${SUBMIT_URL:='http://104.154.54.203/db_default/v4/nts/submitRun'}
+: ${SUBMIT_NAME:="${JOB_NAME-}"}
+
+build get lnt
+build get compiler
+build get test-suite
+
+. "${TASKDIR}"/utils/normalize_compiler.sh
+. "${TASKDIR}"/utils/venv.sh
+. "${TASKDIR}"/utils/venv_lit.sh
+. "${TASKDIR}"/utils/venv_lnt.sh
+
+. "${TASKDIR}"/utils/lnt_test_suite.sh
+. "${TASKDIR}"/utils/lnt_submit.sh
+
+. "${TASKDIR}"/utils/lnt_move_results.sh
+. "${TASKDIR}"/utils/lnt_check_no_errors.sh
Added: zorg/trunk/tasks/repos/clang-stage1-configure-RA.json
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/repos/clang-stage1-configure-RA.json?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/repos/clang-stage1-configure-RA.json (added)
+++ zorg/trunk/tasks/repos/clang-stage1-configure-RA.json Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+{
+ "url": "http://lab.llvm.org:8080/artifacts/clang-stage1-configure-RA",
+ "type": "artifact_server"
+}
Added: zorg/trunk/tasks/repos/compiler.json
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/repos/compiler.json?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/repos/compiler.json (added)
+++ zorg/trunk/tasks/repos/compiler.json Mon Nov 6 19:18:31 2017
@@ -0,0 +1 @@
+link clang-stage1-configure-RA.json
\ No newline at end of file
Propchange: zorg/trunk/tasks/repos/compiler.json
------------------------------------------------------------------------------
svn:special = *
Added: zorg/trunk/tasks/repos/llvm.json
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/repos/llvm.json?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/repos/llvm.json (added)
+++ zorg/trunk/tasks/repos/llvm.json Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+{
+ "type": "git",
+ "url": "https://git.llvm.org/git/llvm.git"
+}
Added: zorg/trunk/tasks/repos/lnt.json
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/repos/lnt.json?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/repos/lnt.json (added)
+++ zorg/trunk/tasks/repos/lnt.json Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+{
+ "url": "https://git.llvm.org/git/lnt.git",
+ "type": "git"
+}
Added: zorg/trunk/tasks/repos/test-suite.json
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/repos/test-suite.json?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/repos/test-suite.json (added)
+++ zorg/trunk/tasks/repos/test-suite.json Mon Nov 6 19:18:31 2017
@@ -0,0 +1,4 @@
+{
+ "url": "https://git.llvm.org/git/test-suite.git",
+ "type": "git"
+}
Added: zorg/trunk/tasks/task
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/task?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/task (added)
+++ zorg/trunk/tasks/task Mon Nov 6 19:18:31 2017
@@ -0,0 +1 @@
+link tasktool/bin/task
\ No newline at end of file
Propchange: zorg/trunk/tasks/task
------------------------------------------------------------------------------
svn:special = *
Added: zorg/trunk/tasks/tasktool/README.md
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/README.md?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/README.md (added)
+++ zorg/trunk/tasks/tasktool/README.md Mon Nov 6 19:18:31 2017
@@ -0,0 +1,184 @@
+# Task Runner #
+
+
+## Introduction ##
+
+The task runner tool executes user defined tasks.
+
+Tasks are recipes for performing typical build and test actions as found on
+most CI systems. The task runner abstracts those so:
+- Tasks can be tracked in a version control repository.
+- Multiple tasks can share common files that come with the repository.
+- Tasks can be run locally without installing a complete CI system.
+- Tasks can be run via ssh on a separate machine without installing a CI system.
+
+Task files describe:
+- Required inputs (source control checkouts, artifacts to download,
+ string parameters)
+- Steps to perform the task
+- The outputs (currently everything the task produced in the `result` folder)
+- Tasks are written as posix shell scripts with some conventions and a well
+ defined environment.
+
+
+## Running ##
+
+### Local ###
+
+Tasks can be run on the local machine without a CI system running. Example:
+```bash
+$ ./task try hello_world3.sh
+```
+
+### One-time Jenkins Build ###
+
+We have a special jenkins job running which accepts task submission for
+one-time execution. Example:
+```bash
+$ ./task submit hello_world3.sh
+```
+
+### Jenkins Job ###
+
+Tasks can be executed as part of a normal jenkins job. This allows to use
+jenkins triggers like version control changes or reporting features like
+sending mails on failures to be used.
+
+- The tool assumes the CI system already performed the source code checkouts.
+ It will therefore merely verify the expected directories exist.
+- Artifacts specified on the commandline (`-a xxx=yyy`) however work as usual.
+
+The corresponding jenkins job should:
+- Check out the repository containing the task definition and task runner into
+ `${WORKSPACE}/config`
+- Checkout all dependent repositories.
+- Invoke the task tool in a jenkins shell step. Example:
+```bash
+config/task jenkinsrun config/hello_world3.sh
+```
+
+### SSH Remote ###
+
+Tasks can be executed on a remote machine via ssh without having a CI system
+installed. Requirements for this to work:
+
+- Make sure you can login to the remote machine via ssh without a password
+- Make sure you can login from the remote to the local machine via ssh
+ without a password.
+- You can then run on a remote machine. Example:
+```bash
+$ ./task sshrun mymachine.local hello_world3.sh
+```
+
+Notes:
+
+- You can use `username at mymachine.local` to switch user names
+- Use `$HOME/.ssh/config` to configure the ssh connection (see ssh docu)
+- The script uses `ssh -A` so ssh keys from your local machine are forwarded
+ and available on the remote machine.
+
+## Task Tool Operation
+
+### Artifact sources ###
+
+The `task` tool decides what inputs are use for the tasks artifact parameters
+before starting/submitting a build.
+
+This is usually done by querying the repository definitions in the `repos`
+directory (and `repos.try` in some modes). For example for an artifact
+parameter called `llvm` we could have a `repos/llvm.json` configuration file
+to query the latest revision of a git repository:
+```json
+{
+ "url": "https://github.com/llvm-mirror/llvm.git",
+ "type": "git",
+ "default_rev": "refs/heads/master"
+}
+```
+
+To query the latest artifact from a directory in the A2 artifact server:
+```json
+{
+ "url": "http://lab.llvm.org:8080/artifacts/clang-stage1-configure-RA",
+ "type": "artifact_server"
+}
+```
+
+You can specify an URL to an archive file on the commandline to override the
+other mechanisms:
+
+```bash
+$ lnt task try -a compiler=http://my.storage.server/my_compiler.tar.bz2
+```
+
+
+## Task Files ##
+
+Tasks files are written as posix shell scripts.
+The following describes the environment in which they are executed.
+
+### Tutorial ###
+
+`hello_world0.sh` up to `hello_world3.sh` and
+`hello_param.sh` are considered tutorials.
+They are simple tasks that are well documented and show more advanced uses with each script.
+
+### Workspace ###
+
+Scripts execute in an empty directory called workspace:
+ - The `WORKSPACE` environment variable contains the name of this directory.
+ - This directory is exclusively to a single run:
+ No other tasks interfere, no files remaining from previous runs.
+ - Files outside the workspace directory should not be modified.
+ There are some exceptions like creating temporary files or writing
+ to `/dev/null`. See `hooks/try.sb` for the sandbox definition.
+
+### Task Parameters ###
+
+A task can have named string and artifact parameters. An artifact is a
+directory conaining files and subdirectories. Typical artifact sources are git
+repositories or archives from artifact storage servers.
+
+Artifacts can be queried and extracted into a workspace directory with the
+same name:
+```bash
+build get ARTIFACT_NAME
+```
+
+To extract an artifact into a directory with a different name use:
+```bash
+build get DIRECTORY_NAME --from=ARTIFACT_NAME
+```
+
+Arguments for string parameters can be queried with:
+```bash
+VARIABLE="$(build get arg PARAMETER_NAME)"
+```
+
+Parameters marked as optional don't need an argument and will then return an
+empty string:
+```bash
+VARIABLE="$(build get arg --optional PARAMETER_NAME)"
+```
+
+Other default values can be supplied with the usual shell tricks:
+```bash
+: ${VARIABLE:='DefaultValue'}
+```
+
+- Abstract artifacts like `refs/heads/master` in a git repository or
+ `clang-stage2-cmake` on an A2 server are resolved to a concrete git revision
+ or a specific URL when the task is started.
+- A task with a set of artifact and parameter inputs is called a build.
+- `task try mytask.sh` resolves artifact inputs and executes a local build.
+- `task submit mytask.sh` resolves artifact inputs and submits a build to
+ a queue on a jenkins server.
+- `task jenkinsrun mytask.sh` may be used to run the task as part of a
+ recurring jenking project.
+
+### Config Artifact ###
+
+The task files are part of a version control repository. When running a task
+this repository is always checked out to a subdirectory named `config`. This
+allows task files to bundle and reference additional files in the repository
+such as helper scripts shared between multiple tasks.
Added: zorg/trunk/tasks/tasktool/bin/build
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/bin/build?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/bin/build (added)
+++ zorg/trunk/tasks/tasktool/bin/build Mon Nov 6 19:18:31 2017
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+import sys
+import os.path
+sys.path = ['%s/..' % os.path.dirname(os.path.realpath(__file__))] + sys.path
+import tasktool.build
+tasktool.build.main()
Propchange: zorg/trunk/tasks/tasktool/bin/build
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/bin/task
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/bin/task?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/bin/task (added)
+++ zorg/trunk/tasks/tasktool/bin/task Mon Nov 6 19:18:31 2017
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+import sys
+import os.path
+sys.path = ['%s/..' % os.path.dirname(os.path.realpath(__file__))] + sys.path
+import tasktool.task
+tasktool.task.main()
Propchange: zorg/trunk/tasks/tasktool/bin/task
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/setup.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/setup.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/setup.py (added)
+++ zorg/trunk/tasks/tasktool/setup.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,29 @@
+from setuptools import setup, find_packages
+
+setup(
+ name='tasktool',
+ version='1.0',
+ author='Matthias Braun',
+ author_email='matze at braunis.de',
+ license='BSD',
+ description='CI Task Runner',
+ classifiers=[
+ 'Development Status :: 5 - Production/Stable',
+ 'Environment :: Console',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: BSD License',
+ 'Operating System :: POSIX',
+ 'Programming Language :: Python',
+ 'Programming Language :: Unix Shell',
+ 'Topic :: Software Development :: Build Tools',
+ 'Topic :: Software Development :: Quality Assurance'
+ 'Topic :: Software Development :: Testing',
+ ],
+ package_data={
+ 'tasktool': ['hooks/*'],
+ },
+ packages=find_packages(),
+ entry_points={
+ 'console_scripts': ['task=tasktool.task:main'],
+ },
+)
Added: zorg/trunk/tasks/tasktool/tasktool/__init__.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/__init__.py?rev=317539&view=auto
==============================================================================
(empty)
Added: zorg/trunk/tasks/tasktool/tasktool/build.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/build.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/build.py (added)
+++ zorg/trunk/tasks/tasktool/tasktool/build.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,121 @@
+from pipes import quote
+import json
+import os
+import shutil
+import subprocess
+import sys
+import utils
+import tasktool.repos as repos
+
+
+def _read_config():
+ try:
+ configfilename = 'buildconfig.json'
+ with open(configfilename) as configfile:
+ config = json.load(configfile)
+ except Exception as e:
+ sys.stderr.write("Could not read buildconfig.json: %s" % e)
+ sys.exit(1)
+ return config
+
+
+def _get_artifact_from_url(config, dest_dir):
+ url = config.get('url')
+ if url is None:
+ sys.stderr.write("Missing URL for '%s'\n" % name)
+ sys.exit(1)
+ tar_cmd = "cd %s ; curl -s %s | tar -x" % (quote(dest_dir), quote(url))
+ utils.check_call(['mkdir', '-p', dest_dir])
+ utils.check_call(tar_cmd, shell=True)
+
+
+def _command_get(args):
+ if len(args) < 1:
+ sys.stderr.write("Expected remote name\n")
+ sys.exit(1)
+ name = args[0]
+
+ config = _read_config().get(name)
+ if config is None:
+ sys.stderr.write("No config for '%s'\n" % name)
+ sys.exit(1)
+
+ type = config['type']
+ repohandler = repos.modules[type]
+ repohandler.get_artifact(config, name)
+
+
+def _command_arg(args):
+ if len(args) == 0:
+ sys.stderr.write("Expected argument name\n")
+ sys.exit(1)
+ argname = args[0]
+ optional = False
+ if args[0] == '--optional':
+ optional = True
+ argname = args[1]
+
+ config = _read_config().get(argname)
+ if config is None:
+ if not optional:
+ sys.stderr.write("No config entry for '%s'\n" % name)
+ sys.exit(1)
+ config = ''
+
+ sys.stdout.write("%s\n" % config)
+
+
+def _command_clean(args):
+ '''This removes all directories not mentioned in the buildconfig.
+ (It's main usage is as part of 'jenkinsrun' command where we assume that
+ all artifacts are already supplied. And everything else is leftovers from
+ previous build)'''
+ config = _read_config()
+ keep = set(['config', 'buildconfig.json', 'run.sh'])
+ for name, kconfig in config.items():
+ if isinstance(kconfig, dict) and kconfig.get('type') == 'existing':
+ keep.add(name)
+ remove_files = set()
+ for filename in os.listdir('.'):
+ if filename not in keep:
+ remove_files.add(filename)
+ for filename in remove_files:
+ sys.stderr.write("...removing '%s'\n" % filename)
+ try:
+ if os.path.isdir(filename):
+ shutil.rmtree(filename)
+ else:
+ os.unlink(filename)
+ except Exception as e:
+ sys.stderr.write("Error: Could not remove '%s': %s\n" %
+ (filename, e))
+ sys.exit(1)
+
+
+def _command_repro_args(args):
+ '''Produces a message on how to reproduce a particular build locally.'''
+ config = _read_config()
+ for name, kconfig in config.items():
+ sys.stdout.write(' \\\n')
+ if isinstance(kconfig, dict):
+ type = kconfig['type']
+ repohandler = repos.modules[type]
+ repro_arg = repohandler.repro_arg(kconfig, name)
+ sys.stdout.write(' %s' % repro_arg)
+ else:
+ sys.stdout.write(' -D %s=%s' % (name, quote(kconfig)))
+ sys.stdout.write('\n')
+
+
+def main():
+ commands = {
+ 'arg': _command_arg,
+ 'clean': _command_clean,
+ 'get': _command_get,
+ 'repro_args': _command_repro_args,
+ }
+ utils.run_subcommand(commands, sys.argv)
+
+
+if __name__ == '__main__':
+ main()
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/check-clean-repo.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/check-clean-repo.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/check-clean-repo.sh (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/check-clean-repo.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,26 @@
+# One of the most common errors is to forget to add files/changes to the config
+# repository. So we stop the process if the git repository shows any untracked
+# files or non-comitted changes.
+# TODO: Add a commandline flag or similar to override/ignore the checks?
+
+# Check that repo is clean
+if ! git diff-index --quiet HEAD --; then
+ echo 1>&2 "Found uncommitted changes, the build probably won't work."
+ echo 1>&2 "Note: Use `git status` to view them."
+ exit 1
+fi
+
+# Check that the taskscript is actually part of the repo
+if ! git ls-files "${TASKSCRIPT}" --error-unmatch > /dev/null; then
+ echo 1>&2 "Task '${TASKSCRIPT}' is not tracked by git."
+ echo 1>&2 "Note: Add and commit the file to the git repository."
+ exit 1
+fi
+
+# Check for untracked files
+if [ -n "$(git ls-files --others --exclude-standard)" ]; then
+ echo 1>&2 "Error: There are untracked files around, the build probably won't work."
+ echo 1>&2 "Note: - Add the files and commit them"
+ echo 1>&2 "Note: - Or add them to \`.gitignore\` or \`.git/info/exclude\`"
+ exit 1
+fi
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/check-config.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/check-config.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/check-config.sh (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/check-config.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,5 @@
+if ! [ -e "${USERDIR}/config" ]; then
+ echo 1>&2 "Error: 'config' file does not exist"
+ echo 1>&2 "Note: Copy 'config.example' to 'config' and edit"
+ exit 1
+fi
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/exec-try-build
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/exec-try-build?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/exec-try-build (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/exec-try-build Mon Nov 6 19:18:31 2017
@@ -0,0 +1,54 @@
+#!/bin/bash
+set -eu
+
+USERDIR="$1"
+BUILDSCRIPT="$2"
+BUILDNAME="$3"
+
+# Import config if present.
+[ -e "${USERDIR}/config" ] && . "${USERDIR}/config"
+
+# Create workspace directory.
+if [ -z ${TRYDIR+x} ]; then
+ TRYDIR="$TMPDIR"
+ if [ -z ${TRYDIR+x} ]; then
+ TRYDIR="/tmp"
+ fi
+fi
+
+export WORKSPACE="${TRYDIR}/${BUILDNAME}"
+mkdir "${TRYDIR}/${BUILDNAME}"
+echo "Workspace: ${WORKSPACE}"
+
+# Work with a minimum PATH
+export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+
+# Clear environment from some typical user-specific settings.
+# TODO: Should we rather start with a blank environment?
+unset CPATH
+unset LIBRARY_PATH
+unset MANPATH
+unset PKG_CONFIG_PATH
+
+LOGFILE="${WORKSPACE}/log.txt"
+SANDBOXPROFILE="$(pwd)/try.sb"
+cd "${WORKSPACE}"
+: ${HOME:=/tmp}
+
+if [ -z "${TASK_TRY_NO_SANDBOX:-}" ]; then
+ /bin/sh "${BUILDSCRIPT}" 2>&1 | tee "$LOGFILE"
+else
+ # Note that sandboxes don't work recursively, so we set TASK_TRY_NO_SANDBOX
+ # for the child.
+ TASK_TRY_NO_SANDBOX=1 sandbox-exec -f "${SANDBOXPROFILE}" -D WORKSPACE="$WORKSPACE" -D HOME="$HOME" /bin/sh "${BUILDSCRIPT}" 2>&1 | tee "$LOGFILE"
+fi
+
+# Gather results
+if [ -d "${WORKSPACE}/result" ]; then
+ mkdir -p "${USERDIR}/results"
+ RESULTFILE="${USERDIR}/results/${BUILDNAME}.tar.xz"
+ cd "${WORKSPACE}/result"
+ shopt -s dotglob nullglob
+ tar -c --xz -f "${RESULTFILE}" *
+ echo 1>&2 "Results written to ${RESULTFILE}"
+fi
Propchange: zorg/trunk/tasks/tasktool/tasktool/hooks/exec-try-build
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/footer_sshrun
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/footer_sshrun?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/footer_sshrun (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/footer_sshrun Mon Nov 6 19:18:31 2017
@@ -0,0 +1,7 @@
+# This file is appended to all `task sshrun` builds. It is intended to clean
+# up the jenkins environment apropriately.
+
+if [ -d "${WORKSPACE}/result" ]; then
+ cd "${WORKSPACE}/result"
+ tar -c --xz * | ssh "${RESULT_HOST}" "cat > '${RESULT_FILE}'"
+fi
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/header
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/header?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/header (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/header Mon Nov 6 19:18:31 2017
@@ -0,0 +1,14 @@
+# This file is prepended to all builds. It is intended to perform some common
+# envornment setup.
+
+# Choose some safer shell execution modes (abort when using uninitialized vars,
+# abort on non-zero exit codes).
+set -eux
+
+# First things first: Get config repository (use plain git for that as we do
+# not have the advanced scripts available yet).
+git clone -n "${config_url}" config
+(cd config ; git checkout -q "${config_rev}")
+
+# Bring the tools into our PATH
+export PATH="${TASKDIR}/bin:$PATH"
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/header_jenkinsrun
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/header_jenkinsrun?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/header_jenkinsrun (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/header_jenkinsrun Mon Nov 6 19:18:31 2017
@@ -0,0 +1,20 @@
+# This file is prepended to all builds. It is intended to perform some common
+# envornment setup.
+
+if [ -z ${WORKSPACE+x} ]; then
+ echo 1>&2 "Error: WORKSPACE environment variable not defined"
+ exit 1
+fi
+if [ ! -d config ]; then
+ echo 1>&2 "Error: $(pwd)/config directory does not exist"
+ exit 1
+fi
+
+# Choose some safer shell execution modes (abort when using uninitialized vars,
+# abort on non-zero exit codes).
+set -eux
+
+# Bring the tools into our PATH
+export PATH="${TASKDIR}/bin:$PATH"
+
+build clean
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/header_sshrun
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/header_sshrun?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/header_sshrun (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/header_sshrun Mon Nov 6 19:18:31 2017
@@ -0,0 +1,21 @@
+# This file is prepended to all `task sshrun` builds. It is intended to clean
+# up the jenkins environment apropriately.
+
+# Check lock file to avoid running multiple tasks at once.
+set -eu
+SSHRUN_LOCKFILE="$HOME/.task_sshrun_lock"
+if [ -e "$SSHRUN_LOCKFILE" ]; then
+ echo 1>&2 "task sshrun: Already a task running on this machine"
+ echo 1>&2 "Note: Remove '${SSHRUN_LOCKFILE}' if this is wrong"
+ exit 1
+fi
+function cleanup_lock {
+ rm -f "${SSHRUN_LOCKFILE}"
+}
+trap cleanup_lock EXIT
+touch "${SSHRUN_LOCKFILE}"
+
+: ${REMOTE_TRYDIR:="$TMPDIR"}
+WORKSPACE="${REMOTE_TRYDIR/#\~/$HOME}/${BUILDNAME}"
+mkdir -p "${WORKSPACE}"
+cd "${WORKSPACE}"
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/header_submit
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/header_submit?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/header_submit (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/header_submit Mon Nov 6 19:18:31 2017
@@ -0,0 +1,20 @@
+# This file is prepended to all `task submit` builds. It is intended to clean
+# up the jenkins environment apropriately.
+
+# Jenkins does not start with a clean workspace, so we clean it ourself
+export PATH="$PATH:/usr/local/bin"
+rm -rf "${WORKSPACE}"
+mkdir "${WORKSPACE}"
+cd "${WORKSPACE}"
+
+# Jobs should not know/rely on jenkins details in submit mode.
+unset JOB_NAME
+unset NODE_NAME
+unset BUILD_ID
+unset BUILD_URL
+unset JENKINS_URL
+unset EXECUTOR_NUMBER
+unset SVN_REVISION
+unset GIT_COMMIT
+unset GIT_URL
+unset GIT_BRANCH
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-common.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-common.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-common.sh (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-common.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,12 @@
+RELATIVE_TASKSCRIPT="$(git ls-tree --full-name --name-only HEAD "${TASKSCRIPT}")"
+TASKDIR="$(dirname "${RELATIVE_TASKSCRIPT}")"
+
+echo "config_url='${CONFIG_URL}'"
+echo "config_rev='${CONFIG_REV}'"
+echo "TASKDIR=\"\${WORKSPACE}/config/${TASKDIR}\""
+tail -n +4 header
+echo "cat > buildconfig.json <<'BUILDCONFIGEOF'"
+cat "${BUILDCONFIG}"
+echo "BUILDCONFIGEOF"
+echo ""
+cat "${TASKSCRIPT}"
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-id.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-id.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-id.sh (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/mk-build-id.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+# By convention produce a build id in the first line. For now use a uuid.
+BUILDID="$(uuidgen | tr -d "-" | tr "[:upper:]" "[:lower:]")"
+echo "buildid='${BUILDID}'"
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-jenkinsrun-build
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/mk-jenkinsrun-build?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/mk-jenkinsrun-build (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/mk-jenkinsrun-build Mon Nov 6 19:18:31 2017
@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+
+USERDIR="$1"
+TASKSCRIPT="$2"
+BUILDCONFIG="$3"
+
+ABSOLUTE_TASKDIR="$(dirname "${TASKSCRIPT}")"
+
+. mk-build-id.sh
+echo "TASKDIR=\"${ABSOLUTE_TASKDIR}\""
+tail -n +4 header_jenkinsrun
+cat "${TASKSCRIPT}"
Propchange: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-jenkinsrun-build
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-sshrun-build
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/mk-sshrun-build?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/mk-sshrun-build (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/mk-sshrun-build Mon Nov 6 19:18:31 2017
@@ -0,0 +1,34 @@
+#!/bin/sh
+set -eu
+
+USERDIR="$1"
+TASKSCRIPT="$2"
+BUILDCONFIG="$3"
+BUILDNAME="$4"
+
+TASK_DIR="$(dirname "${TASKSCRIPT}")"
+export GIT_WORK_TREE="$(git -C "${TASK_DIR}" rev-parse --show-toplevel)"
+GIT_DIR="$(git -C "${TASK_DIR}" rev-parse --git-dir)"
+export GIT_DIR="$(cd "${TASK_DIR}" ; cd "${GIT_DIR}" ; pwd)"
+
+. check-config.sh
+. "${USERDIR}/config"
+
+. check-clean-repo.sh
+
+HOST="$(whoami)@$(hostname -f)"
+CONFIG_URL="${HOST}:${GIT_WORK_TREE}"
+CONFIG_REV="$(git rev-parse HEAD)"
+
+. mk-build-id.sh
+echo "BUILDNAME=\"${BUILDNAME}_${BUILDID}\""
+if [ -n "${REMOTE_TRYDIR+x}" ]; then
+ echo "REMOTE_TRYDIR=\"${REMOTE_TRYDIR}\""
+fi
+tail -n +4 header_sshrun
+. mk-build-common.sh
+
+mkdir -p "${USERDIR}/results"
+echo "RESULT_HOST='${HOST}'"
+echo "RESULT_FILE='${USERDIR}/results/${BUILDNAME}_${BUILDID}.tar.xz'"
+tail -n +4 footer_sshrun
Propchange: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-sshrun-build
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-build
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-build?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-build (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-build Mon Nov 6 19:18:31 2017
@@ -0,0 +1,43 @@
+#!/bin/bash
+set -eu
+
+USERDIR="$1"
+TASKSCRIPT="$2"
+BUILDCONFIG="$3"
+
+TASK_DIR="$(dirname "${TASKSCRIPT}")"
+export GIT_WORK_TREE="$(git -C "${TASK_DIR}" rev-parse --show-toplevel)"
+GIT_DIR="$(git -C "${TASK_DIR}" rev-parse --git-dir)"
+export GIT_DIR="$(cd "${TASK_DIR}" ; cd "${GIT_DIR}" ; pwd)"
+
+. check-config.sh
+. "${USERDIR}/config"
+
+. check-clean-repo.sh
+
+# We need a branch name for remote submission
+BRANCH="$(git symbolic-ref HEAD)"
+BRANCH="${BRANCH##refs/heads/}"
+if test "$(git rev-parse HEAD)" != "$(git rev-parse $BRANCH)"; then
+ echo 1>&2 "HEAD is not at branch for submission"
+ exit 1
+fi
+
+# Make sure the branch is actually pushed to the remote
+if [ -z ${REMOTE+x} ]; then
+ REMOTE="$(git config branch.${BRANCH}.remote)"
+ if [ -z ${REMOTE+x} ]; then
+ REMOTE="origin"
+ fi
+fi
+if [ "$(git rev-parse $REMOTE/$BRANCH)" != "$(git rev-parse $BRANCH)" ]; then
+ echo 1>&2 "Branch $BRANCH is not up to date on remote $REMOTE"
+ exit 1
+fi
+
+CONFIG_URL="$(git remote get-url $REMOTE)"
+CONFIG_REV="$BRANCH"
+
+. mk-build-id.sh
+tail -n +4 header_submit
+. mk-build-common.sh
Propchange: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-build
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-results
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-results?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-results (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-results Mon Nov 6 19:18:31 2017
@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+
+USERDIR="$1"
+BUILDNAME="$2"
+
+. check-config.sh
+. "${USERDIR}/config"
+
+# Produce commands that gather and send the results
+echo ''
+echo '# Create result artifact and upload'
+echo 'if [ -d "${WORKSPACE}/result" ]; then'
+echo ' cd "${WORKSPACE}/result"'
+# TODO: We could be fancier and stream directly to the destination instead of
+# creating a .tar.gz intermediate file...
+echo ' shopt -s dotglob nullglob'
+echo ' tar -c --gzip -f ../result.tar.gz *'
+echo " curl -s -S -f -X POST -F file=@\"\${WORKSPACE}/result.tar.gz\" '${SUBMIT_RESULTS_URL}/${BUILDNAME}.tar.gz'"
+echo 'fi'
Propchange: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-submit-results
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-try-build
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/mk-try-build?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/mk-try-build (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/mk-try-build Mon Nov 6 19:18:31 2017
@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+
+USERDIR="$1"
+TASKSCRIPT="$2"
+BUILDCONFIG="$3"
+
+TASK_DIR="$(dirname "${TASKSCRIPT}")"
+export GIT_WORK_TREE="$(git -C "${TASK_DIR}" rev-parse --show-toplevel)"
+GIT_DIR="$(git -C "${TASK_DIR}" rev-parse --git-dir)"
+export GIT_DIR="$(cd "${TASK_DIR}" ; cd "${GIT_DIR}" ; pwd)"
+
+. check-clean-repo.sh
+
+CONFIG_URL="${GIT_WORK_TREE}"
+CONFIG_REV="$(git rev-parse HEAD)"
+
+. mk-build-id.sh
+. mk-build-common.sh
Propchange: zorg/trunk/tasks/tasktool/tasktool/hooks/mk-try-build
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/sshrun
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/sshrun?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/sshrun (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/sshrun Mon Nov 6 19:18:31 2017
@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu -o pipefail
+
+REMOTE_HOSTNAME="$1"
+BUILDSCRIPT="$2"
+
+# FIXME: Passing the buildscript as a big commandline argument is dangerous:
+# Most shells limit the maximum length of a commandline.
+echo "ssh -A \"${REMOTE_HOSTNAME}\""
+ssh -A "${REMOTE_HOSTNAME}" "$(cat "${BUILDSCRIPT}")"
Propchange: zorg/trunk/tasks/tasktool/tasktool/hooks/sshrun
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/submit
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/submit?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/submit (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/submit Mon Nov 6 19:18:31 2017
@@ -0,0 +1,39 @@
+#!/bin/sh
+set -eu -o pipefail
+
+USERDIR="$1"
+BUILDSCRIPT="$2"
+BUILDNAME="$3"
+
+. check-config.sh
+. "${USERDIR}/config"
+
+CURL_ARGS=" -s -XPOST"
+CURL_ARGS+=" -i"
+CURL_ARGS+=" -u \"${JENKINS_USER}:${JENKINS_TOKEN}\""
+CURL_ARGS+=" \"${JENKINS_URL}/job/${JENKINS_SUBMIT_PROJECT}/buildWithParameters\""
+CURL_ARGS+=" --data-urlencode BUILDSCRIPT@\"$BUILDSCRIPT\""
+CURL_ARGS+=" --data-urlencode BUILDNAME=\"${BUILDNAME}\""
+if [ -n "${JENKINS_SUBMIT_NODE+}" ]; then
+ CURL_ARGS+=" --data-urlencode where=\"${JENKINS_SUBMIT_NODE}\""
+fi
+
+eval curl $CURL_ARGS | grep Location
+
+# TODO: Get a handle on something that allows us to track the build as it goes
+# through jenkins. This is hard to do because:
+# - Jenkins gives us back a handle for the build in the Queue (the "Location"
+# header in the answer)
+# - Once the build is executed the queue handle gets updated with a handle
+# to the actual build. However queue handle becomes invalid after a while.
+# => You have to actively poll/query for the queue item changing into a build to
+# catch it :-/
+#
+# Alternatively our builds are annotated with a unique ID and we set the jenkins
+# build name to that. We could search all the jenkins builds to find a specific
+# ID (probably good enough for a single node, is that efficient when we have
+# to search mutliple or all nodes?).
+
+# For now the user has to search/track himself:
+echo ""
+echo "Note: Build results at ${JENKINS_URL}/job/${JENKINS_SUBMIT_PROJECT}"
Propchange: zorg/trunk/tasks/tasktool/tasktool/hooks/submit
------------------------------------------------------------------------------
svn:executable = *
Added: zorg/trunk/tasks/tasktool/tasktool/hooks/try.sb
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/hooks/try.sb?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/hooks/try.sb (added)
+++ zorg/trunk/tasks/tasktool/tasktool/hooks/try.sb Mon Nov 6 19:18:31 2017
@@ -0,0 +1,33 @@
+;; Sandbox profile for try executions so they can only write/remove stuff
+;; within their workspace directory (TODO: Restrict reading too)
+(version 1)
+(allow default)
+
+(define (home-regex home-relative-regex)
+ (regex (string-append "^" (regex-quote (param "HOME")) home-relative-regex)))
+(define (home-subpath home-relative-subpath)
+ (subpath (string-append (param "HOME") home-relative-subpath)))
+(define (home-literal home-relative-literal)
+ (literal (string-append (param "HOME") home-relative-literal)))
+
+(deny file-write*)
+(allow file-write*
+ (subpath (param "WORKSPACE"))
+ ; temp files/dirs
+ (subpath "/private/tmp")
+ (subpath "/private/var/folders")
+ (subpath "/private/var/tmp")
+ (home-subpath "/Library/Caches/pip")
+ ; for /dev/null, /dev/dtracehelper possibly others
+ (subpath "/dev")
+)
+
+;; TODO: Restrict reading to a minimum. Something like:
+;;(deny file-read*)
+;;(allow file-read*
+;; (subpath "/bin")
+;; (subpath "/sbin")
+;; (subpath "/usr/bin")
+;; (subpath "/usr/local/bin")
+;; (subpath "/Applications/Xcode.app")
+;;)
Added: zorg/trunk/tasks/tasktool/tasktool/repos/__init__.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/repos/__init__.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/repos/__init__.py (added)
+++ zorg/trunk/tasks/tasktool/tasktool/repos/__init__.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,21 @@
+import importlib
+import logging
+import pkgutil
+
+# Load all repository handlers.
+modules = dict()
+_required_functions = ('verify', 'resolve_latest', 'get_artifact',
+ 'repro_arg')
+for importer, modname, ispkg in pkgutil.walk_packages(path=__path__,
+ prefix=__name__+'.'):
+ module = importlib.import_module(modname)
+ bad_module = False
+ for function in _required_functions:
+ if not hasattr(module, function):
+ logging.error('Ignoring %s: No %s function' % function)
+ bad_module = True
+ if bad_module:
+ continue
+ assert modname.startswith('%s.' % __name__)
+ shortname = modname[len('%s.' % __name__):]
+ modules[shortname] = module
Added: zorg/trunk/tasks/tasktool/tasktool/repos/artifact_server.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/repos/artifact_server.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/repos/artifact_server.py (added)
+++ zorg/trunk/tasks/tasktool/tasktool/repos/artifact_server.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,57 @@
+'''
+Get artifact from an artifact server.
+
+The server should present a directory structure over http/https. The leaf
+directories should contain a file called `last_good_build.properties`
+(typically generated by jenkins) containing a line like:
+
+ ARTIFACT=subdir/buz.tar.gz
+
+which is then resolved as the latest version available for this directory.
+'''
+import os
+import sys
+import tasktool.utils as utils
+
+
+def verify(config):
+ if config.get('url') is None:
+ raise Exception("No 'url' specified")
+
+
+def resolve_latest(config):
+ assert(config['type'] == 'artifact_server')
+ url = config['url']
+ if not url.endswith('/'):
+ url += '/'
+ latest_info_file = config.get('latest_info_file',
+ 'last_good_build.properties')
+ latest_url = url + latest_info_file
+ props = utils.check_output(['curl', '-s', latest_url])
+
+ res = None
+ latest_key = config.pop('latest_key', 'ARTIFACT')
+ for line in props.split('\n'):
+ line = line.strip()
+ if line.startswith('%s=' % latest_key):
+ res = line[len(latest_key)+1:]
+ break
+
+ # We usually have something like, try to resolve this to a single URL
+ # URL=https://myserver.com/mybuild/
+ # ARTIFACT=mybuild/blabla.tar.gz
+ dirname = os.path.dirname(res) + "/"
+ full_url = url
+ if full_url.endswith(dirname):
+ full_url = full_url[:-len(dirname)]
+ full_url += res
+ config['url'] = full_url
+ config['type'] = 'url'
+
+
+def get_artifact(config, dest_dir):
+ raise Exception("A2 artifact not resolved to URL?!?")
+
+
+def repro_arg(config, dest_dir):
+ raise Exception("A2 artifact not resolved to URL?!?")
Added: zorg/trunk/tasks/tasktool/tasktool/repos/existing.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/repos/existing.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/repos/existing.py (added)
+++ zorg/trunk/tasks/tasktool/tasktool/repos/existing.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,28 @@
+import sys
+import os.path
+import tasktool.utils as utils
+
+
+def verify(config):
+ pass
+
+
+def resolve_latest(config):
+ pass
+
+
+def get_artifact(config, dest_dir):
+ if not os.path.isdir(dest_dir):
+ sys.stderr.write("Error: Expected directory '%s' is missing\n" %
+ (dest_dir, ))
+ sys.exit(1)
+
+
+def repro_arg(config, dest_dir):
+ if os.path.exists('%s/.git' % dest_dir):
+ rev = utils.check_output(['git', '--git-dir=%s/.git' % dest_dir,
+ 'rev-parse', 'HEAD'])
+ rev = rev.strip()
+ return '-r %s=%s' % (dest_dir, rev)
+ else:
+ return "!Unknown repository type in '%s'" % dest_dir
Added: zorg/trunk/tasks/tasktool/tasktool/repos/git.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/repos/git.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/repos/git.py (added)
+++ zorg/trunk/tasks/tasktool/tasktool/repos/git.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,61 @@
+import sys
+import tasktool.utils as utils
+
+
+def _git_ls_remote(url, ref):
+ refs = utils.check_output(['git', 'ls-remote', url, ref])
+ revs = {}
+ for line in refs.split("\n"):
+ line = line.strip()
+ if line == "":
+ continue
+ rev, _, name = line.partition('\t')
+ revs[name] = rev
+
+ return revs
+
+
+def verify(config):
+ if config.get('url') is None:
+ raise Exception("No 'url' specified")
+
+
+def resolve_latest(config):
+ assert(config['type'] == 'git')
+ rev = config.get('rev')
+ default_rev = config.pop('default_rev', None)
+ if default_rev is None:
+ default_rev = "refs/heads/master"
+ if rev is not None:
+ return
+ url = config['url']
+ revs = _git_ls_remote(url, default_rev)
+ # Look for an exact match
+ if default_rev is not None:
+ rev = revs.get(default_rev, None)
+ # Otherwise hope that we only have a single match.
+ if rev is None:
+ if len(revs) == 0:
+ raise Exception("No refs matching '%s' found for "
+ "repository '%s'\n" % (default_rev, url))
+ if len(revs) > 1:
+ raise Exception("Found multiple refs matching '%s' for "
+ "repository '%s': %s\n" %
+ (default_rev, url, ", ".join(revs.keys())))
+ rev = revs.values()[0]
+ config['rev'] = rev
+
+
+def get_artifact(config, dest_dir):
+ url = config['url']
+ rev = config['rev']
+ # Note: -s is slightly dangerous for long running builds on the local
+ # machine where the user changes the reference repo. But it's also a good
+ # bit faster.
+ utils.check_call(['git', 'clone', '-q', '-n', '-s', url, dest_dir])
+ utils.check_call(['git', '--git-dir=%s/.git' % dest_dir,
+ '--work-tree=%s' % dest_dir, 'checkout', '-q', rev])
+
+
+def repro_arg(config, dest_dir):
+ return '-r %s=%s' % (dest_dir, config['rev'])
Added: zorg/trunk/tasks/tasktool/tasktool/repos/url.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/repos/url.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/repos/url.py (added)
+++ zorg/trunk/tasks/tasktool/tasktool/repos/url.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,24 @@
+from pipes import quote
+import sys
+import tasktool.utils as utils
+import logging
+
+
+def verify(config):
+ if config.get('url') is None:
+ raise Exception("No 'url' specified")
+
+
+def resolve_latest(config):
+ pass
+
+
+def get_artifact(config, dest_dir):
+ url = config['url']
+ untar_cmd = "cd %s ; curl -s %s | tar -x" % (quote(dest_dir), quote(url))
+ utils.check_call(['mkdir', '-p', dest_dir])
+ utils.check_call(untar_cmd, shell=True)
+
+
+def repro_arg(config, dest_dir):
+ return '-a %s=%s' % (dest_dir, quote(config['url']))
Added: zorg/trunk/tasks/tasktool/tasktool/task.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/task.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/task.py (added)
+++ zorg/trunk/tasks/tasktool/tasktool/task.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,443 @@
+'''
+Task runner.
+'''
+import argparse
+import copy
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import utils
+import repos
+
+
+_userdir = os.path.abspath('.')
+_hooksdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'hooks'))
+_artifact_input_regex = \
+ re.compile(r'\s*#?\s*build\s+get\s+([a-zA-Z_\-0-9]+)')
+_artifact_input_regex_ex = \
+ re.compile(r'\s*#?\s*build\s+get\s+([a-zA-Z_\-0-9]+)\s+--from=([a-zA-Z_\-0-9]+)')
+_artifact_param_regex = \
+ re.compile(r'build\s+arg\s+([a-zA-Z_\-0-9]+)\s*')
+_artifact_optional_param_regex = \
+ re.compile(r'build\s+arg\s+--optional\s+([a-zA-Z_\-0-9]+)\s*')
+
+
+def _determine_task_inputs(taskfile):
+ artifacts = dict()
+ parameters = set()
+ for line in taskfile:
+ m = _artifact_input_regex_ex.search(line)
+ if m:
+ name = m.group(1)
+ repo = m.group(2)
+ artifacts[name] = repo
+ continue
+ m = _artifact_input_regex.search(line)
+ if m:
+ name = m.group(1)
+ artifacts[name] = name
+ continue
+ m = _artifact_optional_param_regex.search(line)
+ if m:
+ name = m.group(1)
+ optional = True
+ parameter = (name, optional)
+ parameters.add(parameter)
+ continue
+ m = _artifact_param_regex.search(line)
+ if m:
+ name = m.group(1)
+ optional = False
+ parameter = (name, optional)
+ parameters.add(parameter)
+ continue
+
+ if 'config' in artifacts:
+ sys.stderr.write("%s: Artifact name 'config' is reserved\n" % taskfile)
+ sys.exit(1)
+
+ return (artifacts, list(parameters))
+
+
+def _get_configfilename(taskfilename):
+ configname = taskfilename
+ if configname.endswith('.sh'):
+ configname = configname[:-3]
+ configname += '.json'
+ return configname
+
+
+def _find_repo_config(taskdir, reponame, extra_searchpath=[]):
+ repo_searchpath = extra_searchpath + ['repos']
+
+ for path in repo_searchpath:
+ full_path = os.path.join(taskdir, path, reponame+'.json')
+ if os.path.exists(full_path):
+ return full_path
+ sys.stderr.write("There is no configuration for repository '%s'\n" %
+ reponame)
+ sys.stderr.write("Note: Searchpath: %s\n" % ", ".join(repo_searchpath))
+ sys.exit(1)
+
+
+def _read_repo_config(taskdir, reponame, extra_searchpath):
+ configfile = _find_repo_config(taskdir, reponame, extra_searchpath)
+ with open(configfile) as file:
+ try:
+ repoconfig = json.load(file)
+ except ValueError as e:
+ sys.stderr.write("%s: error: %s\n" % (configfile, e))
+ sys.exit(1)
+
+ copyfrom = repoconfig.get('copyfrom')
+ if copyfrom is not None:
+ merged_config = repoconfig
+ del merged_config['copyfrom']
+ copyfrom_config = _read_repo_config(taskdir, copyfrom, extra_searchpath)
+ for key, val in copyfrom_config.items():
+ merged_config[key] = val
+ repoconfig = merged_config
+
+ # Check repoconfig for errors.
+ type = repoconfig.get('type')
+ if type is None:
+ sys.stderr.write("No type specified in repo config '%s'\n" %
+ configfile)
+ sys.exit(1)
+ repohandler = repos.modules.get(type)
+ if repohandler is None:
+ sys.stderr.write("Unknown type '%s' in repo config '%s'\n" %
+ (type, configfile))
+ sys.exit(1)
+ try:
+ repohandler.verify(repoconfig)
+ except Exception as e:
+ sys.stderr.write("Invalid repo config '%s': %s\n" %
+ (configfile, e))
+ sys.exit(1)
+ return repoconfig
+
+
+def _make_task_argparser(command_name, debughelper_mode=False,
+ hostname_arg=False):
+ p = argparse.ArgumentParser(prog=('task %s' % command_name))
+ p.set_defaults(name=None)
+ p.set_defaults(local=False)
+ p.set_defaults(existing=False)
+ p.set_defaults(rewrite_local=False)
+ if hostname_arg:
+ p.add_argument('hostname')
+ p.add_argument('task')
+ p.add_argument('-a', '--artifact', action='append', default=[],
+ dest='artifacts')
+ p.add_argument('-r', '--ref', action='append', default=[], dest='refs')
+ p.add_argument('-D', '--define', action='append', default=[], dest='defs')
+ p.add_argument('-v', '--verbose', action='store_true', default=False)
+ if debughelper_mode:
+ p.add_argument('-l', '--local', action='store_true')
+ p.add_argument('-e', '--existing', action='store_true')
+ p.add_argument('-L', '--rewrite-local', action='store_true')
+ else:
+ p.add_argument('-n', '--name')
+ return p
+
+
+def _rewrite_local_git_urls(buildconfig):
+ '''
+ Prefix all git repository urls in buildconfig that start with a slash
+ (they reference local files) with a prefix of the local machine/user.
+ '''
+ hostname = utils.check_output(['hostname', '-f']).strip()
+ user = utils.check_output(['whoami']).strip()
+ for name, config in buildconfig.items():
+ if not isinstance(config, dict):
+ continue
+ url = config.get('url')
+ if url is not None and url.startswith('/'):
+ config['url'] = '%s@%s:%s' % (user, hostname, url)
+
+
+class BuildConfig(object):
+ def __init__(self, taskfilename, taskname, config):
+ self.taskfilename = taskfilename
+ self.taskname = taskname
+ self.config = config
+
+
+def _make_buildconfig(argconfig):
+ if argconfig.verbose:
+ utils.verbose = True
+ taskfilename = os.path.abspath(argconfig.task)
+ taskname = argconfig.name
+ if taskname is None:
+ taskname = os.path.basename(taskfilename).partition('.')[0]
+
+ try:
+ with open(taskfilename) as taskfile:
+ artifact_parameters, parameters = _determine_task_inputs(taskfile)
+ except IOError as e:
+ sys.stderr.write("%s\n" % (e,))
+ sys.exit(1)
+
+ # Resolve arguments
+ buildconfig = dict()
+ mandatory_parameters = set()
+ optional_parameters = set()
+ for parameter, optional in parameters:
+ if optional:
+ optional_parameters.add(parameter)
+ else:
+ mandatory_parameters.add(parameter)
+ for d in argconfig.defs:
+ name, eq, val = d.partition('=')
+ if eq != '=':
+ sys.stderr.write("Expected 'key=value' for -D argument\n")
+ sys.exit(1)
+ if name not in mandatory_parameters and \
+ name not in optional_parameters:
+ sys.stderr.write("Warning: task does not have parameter '%s'\n" %
+ name)
+ buildconfig[name] = val
+ mandatory_parameters.discard(name)
+ if len(mandatory_parameters) > 0:
+ for param in mandatory_parameters:
+ sys.stderr.write("Error: No value for mandatory parameter '%s'\n" %
+ param)
+ sys.stderr.write("Note: Use the `-D parameter=value` option\n")
+ sys.exit(1)
+
+ # Resolve artifact inputs
+ extra_searchpath = ["repos.try"] if argconfig.local else []
+ taskdir = os.path.dirname(taskfilename)
+ repo_overrides = {}
+ for i in argconfig.artifacts:
+ name, eq, val = i.partition('=')
+ if eq != '=':
+ sys.stderr.write("Expected 'name=url' for -i argument\n")
+ sys.exit(1)
+ if name not in artifact_parameters:
+ sys.stderr.write("Warning: task does not have input '%s'\n" % name)
+ artifact_parameters[name] = name
+ if '://' not in val:
+ # TODO: Support local dirs...
+ sys.stderr.write("Expected URL for input '%s'\n" % name)
+ sys.exit(1)
+ repo_overrides[name] = {'type': 'url', 'url': val}
+ for i in argconfig.refs:
+ name, eq, val = i.partition('=')
+ if eq != '=':
+ sys.stderr.write("Expected 'repo=ref' for -r argument\n")
+ sys.exit(1)
+ if name not in artifact_parameters:
+ sys.stderr.write("Warning: task does not have input '%s'\n" % name)
+ continue
+ reponame = artifact_parameters[name]
+ repoconfig = _read_repo_config(taskdir, reponame, extra_searchpath)
+ type = repoconfig['type']
+ if type == 'git':
+ repoconfig['rev'] = val
+ else:
+ sys.stderr.write("Cannot override revision of repo '%s'\n" % name)
+ repo_overrides[name] = repoconfig
+
+ for name, reponame in artifact_parameters.items():
+ repoconfig = repo_overrides.get(name)
+ if repoconfig is None:
+ if argconfig.existing:
+ buildconfig[name] = {'type': 'existing'}
+ continue
+ repoconfig = _read_repo_config(taskdir, reponame, extra_searchpath)
+ type = repoconfig['type']
+ repohandler = repos.modules.get(type)
+ try:
+ repohandler.resolve_latest(repoconfig)
+ except Exception as e:
+ sys.stderr.write("While resolving %s:\n" % reponame)
+ sys.stderr.write("%s\n" % e)
+ sys.exit(1)
+ buildconfig[name] = repoconfig
+
+ if argconfig.rewrite_local:
+ _rewrite_local_git_urls(buildconfig)
+
+ return BuildConfig(taskfilename, taskname, buildconfig)
+
+
+def _extract_buildid(build):
+ '''
+ Extract buildid from buildscript. By convention the first line of a
+ complete buildscript has to start with `buildid='`
+ '''
+ firstline = build.splitlines()[0]
+ if not firstline.startswith("buildid='") or not firstline.endswith("'"):
+ sys.stderr.write(
+ "error: build malformed (must start with buildid='...')\n")
+ sys.exit(1)
+ return firstline[9:-1]
+
+
+def _make_buildscript(hook, buildconfig, keep_buildconfig=False):
+ if not keep_buildconfig:
+ buildconfig_file = tempfile.NamedTemporaryFile(prefix='buildconfig_',
+ delete=False)
+ buildconfig_filename = buildconfig_file.name
+ else:
+ buildconfig_filename = 'buildconfig.json'
+ buildconfig_file = open(buildconfig_filename, 'w')
+
+ with buildconfig_file:
+ buildconfig_file.write(json.dumps(buildconfig.config, indent=2))
+ buildconfig_file.write('\n')
+ buildconfig_file.close()
+ build = utils.check_output([hook, _userdir, buildconfig.taskfilename,
+ buildconfig_filename,
+ buildconfig.taskname], cwd=_hooksdir)
+ if not keep_buildconfig:
+ os.unlink(buildconfig_filename)
+
+ # Extract buildid and create buildname
+ buildid = _extract_buildid(build)
+ buildname = buildconfig.taskname + "_" + buildid
+
+ return build, buildname
+
+
+def _mk_submit_results(buildname):
+ return utils.check_output(['./mk-submit-results', _userdir, buildname],
+ cwd=_hooksdir)
+
+
+def _command_try(args):
+ '''Execute task locally.'''
+ p = _make_task_argparser('try')
+ argconfig = p.parse_args(args)
+ argconfig.local = True
+ buildconfig = _make_buildconfig(argconfig)
+ build, buildname = _make_buildscript('./mk-try-build', buildconfig)
+
+ with tempfile.NamedTemporaryFile(delete=False) as tempf:
+ tempf.write(build)
+ tempf.close()
+ utils.check_call(['./exec-try-build', _userdir, tempf.name, buildname],
+ cwd=_hooksdir)
+ os.unlink(tempf.name)
+
+
+def _command_submit(args):
+ '''Submit task to jenkins OneOff job.'''
+ p = _make_task_argparser('submit')
+ argconfig = p.parse_args(args)
+ buildconfig = _make_buildconfig(argconfig)
+ build, buildname = _make_buildscript('./mk-submit-build', buildconfig)
+
+ build += _mk_submit_results(buildname)
+
+ with tempfile.NamedTemporaryFile(delete=False) as tempf:
+ tempf.write(build)
+ tempf.close()
+ utils.check_call(['./submit', _userdir, tempf.name, buildname],
+ cwd=_hooksdir)
+ os.unlink(tempf.name)
+
+
+def _command_jenkinsrun(args):
+ '''Run task as part of a jenkins job.'''
+ p = _make_task_argparser('jenkinsrun')
+ p.add_argument('-s', '--submit', action='store_true', default=False,
+ help='Submit results to artifact storage at end of task')
+ argconfig = p.parse_args(args)
+ argconfig.existing = True
+ buildconfig = _make_buildconfig(argconfig)
+ build, buildname = _make_buildscript('./mk-jenkinsrun-build', buildconfig,
+ keep_buildconfig=True)
+
+ if argconfig.submit:
+ build += _mk_submit_results(buildname)
+
+ with open("run.sh", "w") as runfile:
+ runfile.write(build)
+
+ retcode = utils.call(['/bin/sh', 'run.sh'])
+ if retcode != 0:
+ sys.stdout.write("*Build failed!* (return code %s)\n" % retcode)
+ sys.stdout.flush()
+ taskdir = os.path.dirname(buildconfig.taskfilename)
+ repro_script = os.path.join(taskdir, 'repro_message.sh')
+ if os.access(repro_script, os.X_OK):
+ utils.check_call([repro_script, _userdir,
+ buildconfig.taskfilename])
+ sys.exit(retcode)
+
+
+def _command_sshrun(args):
+ '''Run task by logging into a remote machine with ssh.'''
+ p = _make_task_argparser('sshrun', hostname_arg=True)
+ argconfig = p.parse_args(args)
+ argconfig.local = True
+ argconfig.rewrite_local = True
+ buildconfig = _make_buildconfig(argconfig)
+ build, buildname = _make_buildscript('./mk-sshrun-build', buildconfig)
+
+ run_file = tempfile.NamedTemporaryFile(prefix=buildname, delete=False)
+ with run_file:
+ run_file.write(build)
+ run_file.close()
+ try:
+ utils.check_call(['./sshrun', argconfig.hostname, run_file.name],
+ cwd=_hooksdir)
+ finally:
+ os.unlink(run_file.name)
+
+
+def _command_resolve(args):
+ '''Print artifact resolution results. (debug helper)'''
+ p = _make_task_argparser('resolve', debughelper_mode=True)
+ argconfig = p.parse_args(args)
+ buildconfig = _make_buildconfig(argconfig)
+
+ for name, config in sorted(buildconfig.config.items()):
+ if isinstance(config, dict):
+ url = config.get('url', '')
+ line = "%-15s\t%-30s" % (name, url)
+ rev = config.get('rev')
+ if rev is not None:
+ line += " " + rev
+ sys.stdout.write("%s\n" % line)
+ else:
+ sys.stdout.write("%-15s\t%s\n" % (name, config))
+
+
+def _command_buildconfig(args):
+ '''Produce buildconfig. (debug helper)'''
+ p = _make_task_argparser('buildconfig', debughelper_mode=True)
+ argconfig = p.parse_args(args)
+ buildconfig = _make_buildconfig(argconfig)
+
+ json.dump(buildconfig.config, sys.stdout, indent=2, sort_keys=True)
+ sys.stdout.write('\n')
+
+
+def main():
+ argv = sys.argv
+ # Allow overriding _hooksdir for testing.
+ if len(argv) > 1 and argv[1].startswith("--hooks-dir="):
+ global _hooksdir
+ _hooksdir = argv[1].split('=', 1)[1]
+ del argv[1]
+
+ commands = {
+ 'buildconfig': _command_buildconfig,
+ 'jenkinsrun': _command_jenkinsrun,
+ 'resolve': _command_resolve,
+ 'sshrun': _command_sshrun,
+ 'submit': _command_submit,
+ 'try': _command_try,
+ }
+ utils.run_subcommand(commands, argv, docstring=__doc__)
+
+
+if __name__ == '__main__':
+ main()
Added: zorg/trunk/tasks/tasktool/tasktool/utils.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/tasktool/tasktool/utils.py?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/tasktool/tasktool/utils.py (added)
+++ zorg/trunk/tasks/tasktool/tasktool/utils.py Mon Nov 6 19:18:31 2017
@@ -0,0 +1,89 @@
+from pipes import quote
+import inspect
+import os
+import subprocess
+import sys
+import textwrap
+
+verbose = False
+
+
+def _logged_catched(func, commandline, *args, **kwargs):
+ commandline_string = commandline
+ if isinstance(commandline, list) or isinstance(commandline, tuple):
+ commandline_string = " ".join([quote(arg) for arg in commandline])
+
+ if verbose:
+ cwd = kwargs.get('cwd', None)
+ if cwd is not None:
+ sys.stderr.write("In Directory: %s\n" % cwd)
+ sys.stderr.write("$ %s\n" % commandline_string)
+ try:
+ return func(commandline, *args, **kwargs)
+ except subprocess.CalledProcessError as e:
+ sys.stderr.write("Error while executing: $ %s\n" %
+ commandline_string)
+ if e.output is not None:
+ sys.stderr.write(str(e.output))
+ sys.exit(e.returncode)
+ except OSError as e:
+ sys.stderr.write("Error while executing $ %s\n" % commandline_string)
+ sys.stderr.write("Error: %s\n" % e)
+ sys.exit(1)
+
+
+def check_call(commandline, *args, **kwargs):
+ """Opinionated version of subprocess.check_call: Logs commandline in
+ verbose mode, and for returncode != 0 print a message and exit the
+ application."""
+ return _logged_catched(subprocess.check_call, commandline, *args, **kwargs)
+
+
+def check_output(commandline, *args, **kwargs):
+ """Opinionated version of subprocess.check_output: Logs commandline in
+ verbose mode, and for returncode != 0 print a message and exit the
+ application."""
+ return _logged_catched(subprocess.check_output, commandline, *args,
+ **kwargs)
+
+
+def call(commandline, *args, **kwargs):
+ """Opinionated version of subprocess.check_output: Logs commandline in
+ verbose mode, and exit with message the if command does not exist."""
+ return _logged_catched(subprocess.call, commandline, *args, **kwargs)
+
+
+def _print_help(commands, argv, docstring):
+ if docstring:
+ description = inspect.cleandoc(docstring)
+ sys.stderr.write(description)
+ sys.stderr.write("\n\n")
+ sys.stderr.write("Usage:\n")
+ twc = textwrap.TextWrapper(expand_tabs=True, initial_indent=' ',
+ subsequent_indent=' ', width=78)
+ for name, func in sorted(commands.items(), key=lambda x: x[0]):
+ sys.stderr.write(" %s %s [arg]...\n" %
+ (os.path.basename(argv[0]), name))
+ if func.__doc__:
+ docstring = inspect.cleandoc(func.__doc__)
+ sys.stderr.write('\n'.join(twc.wrap(docstring)) + '\n')
+
+
+def run_subcommand(commands, argv, docstring=None):
+ if len(argv) < 2:
+ _print_help(commands, argv, docstring)
+ sys.stderr.write("\nError: No command specified!\n")
+ sys.exit(1)
+
+ commandname = argv[1]
+ if commandname == 'help' or commandname == '--help':
+ _print_help(commands, argv, docstring)
+ sys.exit(0)
+
+ command = commands.get(argv[1])
+ if command is None:
+ sys.stderr.write("Error: Unknown command '%s'\n" % argv[1])
+ sys.exit(1)
+
+ args = argv[2:]
+ command(args)
Added: zorg/trunk/tasks/test-suite-verify-machineinstrs.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/test-suite-verify-machineinstrs.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/test-suite-verify-machineinstrs.sh (added)
+++ zorg/trunk/tasks/test-suite-verify-machineinstrs.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,13 @@
+CMAKE_FLAGS="$(build arg --optional CMAKE_FLAGS)"
+
+build get compiler
+build get test-suite
+
+. "${TASKDIR}"/utils/normalize_compiler.sh
+. "${TASKDIR}"/utils/venv.sh
+. "${TASKDIR}"/utils/venv_lit.sh
+
+TEST_SUITE_CMAKE_FLAGS+=" -C ${TASKDIR}/cmake/caches/verify-machineinstrs.cmake"
+TEST_SUITE_CMAKE_FLAGS+=" ${CMAKE_FLAGS}"
+TEST_SUITE_NINJA_FLAGS+=" -v -k0"
+. "${TASKDIR}"/utils/test_suite_compile.sh
Added: zorg/trunk/tasks/utils/lnt_check_no_errors.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/lnt_check_no_errors.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/lnt_check_no_errors.sh (added)
+++ zorg/trunk/tasks/utils/lnt_check_no_errors.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,10 @@
+echo "@@@ LNT Check No Erros @@@"
+
+# Usually the LNT steps do not report errors with an exitcode because we want
+# to continue running, sending mails, etc. even if some of the benchmarks
+# failed. However putting this at the end of a job ensure we do report a
+# problem after all and mark the build as failed.
+
+lnt check-no-errors "${WORKSPACE}/lnt-sandbox/report.json"
+
+echo "@@@@@@@"
Added: zorg/trunk/tasks/utils/lnt_move_results.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/lnt_move_results.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/lnt_move_results.sh (added)
+++ zorg/trunk/tasks/utils/lnt_move_results.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,8 @@
+echo "@@@ Move LNT Results @@@"
+
+mkdir -p result
+test -e lnt-sandbox/build/test.log && cp -p lnt-sandbox/build/test.log result/
+test -e lnt-sandbox/build/report.json && cp -p lnt-sandbox/build/report.json result/
+test -e lnt-sandbox/build/test-results.xunit.xml && cp -p lnt-sandbox/build/test-results.xunit.xml result/
+
+echo "@@@@@@@"
Added: zorg/trunk/tasks/utils/lnt_submit.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/lnt_submit.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/lnt_submit.sh (added)
+++ zorg/trunk/tasks/utils/lnt_submit.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,11 @@
+echo "@@@ LNT Submit @@@"
+
+if [ -n "${SUBMIT_URL:=}" -a -n "${SUBMIT_NAME:=}" ]; then
+ LNT_RESULT_URL="$(lnt submit "${SUBMIT_URL}" "${WORKSPACE}/lnt-sandbox/report.json")"
+ # Jenkins builds look for this message:
+ echo "Results available at: ${LNT_RESULT_URL}"
+else
+ echo 1>&2 "Skipping submission: SUBMIT_URL/SUBMIT_NAME not defined"
+fi
+
+echo "@@@@@@@"
Added: zorg/trunk/tasks/utils/lnt_test_suite.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/lnt_test_suite.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/lnt_test_suite.sh (added)
+++ zorg/trunk/tasks/utils/lnt_test_suite.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,25 @@
+echo "@@@ LNT @@@"
+
+mkdir "lnt-sandbox"
+cd "lnt-sandbox"
+
+LNT_FLAGS+=" --sandbox ${WORKSPACE}/lnt-sandbox"
+LNT_FLAGS+=" --no-timestamp"
+LNT_FLAGS+=" --use-lit=lit"
+LNT_FLAGS+=" --cc ${WORKSPACE}/compiler/bin/clang"
+LNT_FLAGS+=" --cxx ${WORKSPACE}/compiler/bin/clang++"
+LNT_FLAGS+=" --test-suite=${WORKSPACE}/test-suite"
+LNT_FLAGS+=" --cmake-define TEST_SUITE_BENCHMARKING_ONLY=On"
+LNT_FLAGS+=" --output \"${WORKSPACE}/lnt-sandbox/report.json\""
+if [ -n "${SUBMIT_NAME:=}" ]; then
+ # Use jenkins job name as submission name
+ LNT_FLAGS+=" --no-auto-name '${SUBMIT_NAME}'"
+ if test -n "${SUBMIT_ORDER:=}"; then
+ LNT_FLAGS+=" --run-order \"${SUBMIT_ORDER}\""
+ fi
+fi
+
+eval lnt runtest test-suite ${LNT_FLAGS}
+
+cd "${WORKSPACE}"
+echo "@@@@@@"
Added: zorg/trunk/tasks/utils/normalize_compiler.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/normalize_compiler.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/normalize_compiler.sh (added)
+++ zorg/trunk/tasks/utils/normalize_compiler.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,28 @@
+# Get rid of the overly deep directory structures in the compiler artifacts
+#
+# After this script has run you should have $WORKSPACE/compiler/bin/clang etc.
+cd "compiler"
+# For the people with full xcode paths
+if [ -d "Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr" ]; then
+ if [ -d "usr" ]; then
+ rm -rf usr
+ fi
+ mv Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/* .
+ rm -rf Applications
+elif [ -d */bin ]; then
+# For people that have exactly 1 toplevel directory in their artifact
+# */bin/clang -> bin/clang
+ dir=$(dirname */bin)
+ for subdir in "$dir"/*
+ do
+ subdir_basename=$(basename $subdir)
+ if [ -d $subdir_basename ]; then
+ rm -rf $subdir_basename
+ fi
+ mv "$dir"/$subdir_basename .
+ done
+fi
+
+# Make sure there is a clang now
+cd "${WORKSPACE}"
+${WORKSPACE}/compiler/bin/clang -v
Added: zorg/trunk/tasks/utils/pip_install.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/pip_install.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/pip_install.sh (added)
+++ zorg/trunk/tasks/utils/pip_install.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+# This script mainly exists so that internal users can add --extra-index-url
+# arguments to pip.
+pip install "$@"
Added: zorg/trunk/tasks/utils/test_suite_compile.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/test_suite_compile.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/test_suite_compile.sh (added)
+++ zorg/trunk/tasks/utils/test_suite_compile.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,36 @@
+echo "@@@ Test-Suite Compile @@@"
+rm -rf "test-suite-build"
+mkdir -p "test-suite-build"
+cd "test-suite-build"
+TEST_SUITE_CMAKE_FLAGS+="" # Make sure variable is defined
+# Note that we prepend CMAKE_C_COMPILER to ensure it is set before cache files
+# (-C flags) are executed.
+TEST_SUITE_CMAKE_FLAGS="-DCMAKE_C_COMPILER=\"${WORKSPACE}/compiler/bin/clang\" ${TEST_SUITE_CMAKE_FLAGS}"
+
+# Default value
+: ${TEST_SUITE_PGO_BUILD:=}
+
+TEST_SUITE_CMAKE_FLAGS+=" -GNinja"
+TEST_SUITE_CMAKE_FLAGS+=" ${WORKSPACE}/test-suite"
+if [ "${TEST_SUITE_PGO_BUILD}" = "1" ]; then
+ TEST_SUITE_CMAKE_FLAGS+=" -DTEST_SUITE_RUN_TYPE=train"
+ TEST_SUITE_CMAKE_FLAGS+=" -DTEST_SUITE_PROFILE_GENERATE=On"
+fi
+eval cmake ${TEST_SUITE_CMAKE_FLAGS}
+
+TEST_SUITE_NINJA_FLAGS+="" # Make sure variable is defined
+eval ninja "${TEST_SUITE_NINJA_FLAGS}"
+
+if [ "${TEST_SUITE_PGO_BUILD}" = "1" ]; then
+ if [ -z ${TEST_SUITE_LIT_TARGETS+x} ]; then
+ TEST_SUITE_LIT_TARGETS="."
+ fi
+ TEST_SUITE_LIT_FLAGS+="" # make sure variable is defined
+ eval lit -j1 ${TEST_SUITE_LIT_FLAGS} ${TEST_SUITE_LIT_TARGETS}
+ # Rerun cmake with flipped profile flags
+ eval cmake -DTEST_SUITE_PROFILE_GENERATE=Off -DTEST_SUITE_PROFILE_USE=On -DTEST_SUITE_RUN_TYPE=ref .
+ eval ninja "${TEST_SUITE_NINJA_FLAGS}"
+fi
+
+cd "${WORKSPACE}"
+echo "@@@@@@"
Added: zorg/trunk/tasks/utils/venv.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/venv.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/venv.sh (added)
+++ zorg/trunk/tasks/utils/venv.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,6 @@
+echo "@@@ LNT VirtualEnv @@@"
+/usr/local/bin/virtualenv venv
+set +u
+. venv/bin/activate
+set -u
+echo "@@@@@@@"
Added: zorg/trunk/tasks/utils/venv_lit.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/venv_lit.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/venv_lit.sh (added)
+++ zorg/trunk/tasks/utils/venv_lit.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,3 @@
+echo "@@@ Install VirtualEnv Lit @@@"
+. "${TASKDIR}"/utils/pip_install.sh svn+http://llvm.org/svn/llvm-project/llvm/trunk/utils/lit/
+echo "@@@@@@"
Added: zorg/trunk/tasks/utils/venv_lnt.sh
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/tasks/utils/venv_lnt.sh?rev=317539&view=auto
==============================================================================
--- zorg/trunk/tasks/utils/venv_lnt.sh (added)
+++ zorg/trunk/tasks/utils/venv_lnt.sh Mon Nov 6 19:18:31 2017
@@ -0,0 +1,7 @@
+echo "@@@ Install VirtualEnv LNT @@@"
+# It seems that lnt/setup.py does not utilize the python/wheel package caches.
+# Not sure why, we work around the problem by installing the requirements
+# first:
+. "${TASKDIR}"/utils/pip_install.sh -r lnt/requirements.client.txt
+python lnt/setup.py develop
+echo "@@@@@@"
More information about the llvm-commits
mailing list