[llvm] r286123 - Add some facilities to work with a git monorepo (experimental setup)
Mehdi Amini via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 7 09:40:29 PST 2016
Author: mehdi_amini
Date: Mon Nov 7 11:40:28 2016
New Revision: 286123
URL: http://llvm.org/viewvc/llvm-project?rev=286123&view=rev
Log:
Add some facilities to work with a git monorepo (experimental setup)
Summary:
Some changes are made to cmake, especially the addition of a new
LLVM_ENABLE_PROJECTS option that makes the build system aware of
the monorepo directory structure.
Also a new script is added in llvm/utils/git-svn/. When present in
the $PATH, it enables a `git llvm` command. It is providing at this
point only the ability to push from the git monorepo: `git llvm push`.
It is intended to evolves with more features, for instance I plan on
features like `git llvm show r284955` to help working with sequential
revision numbers.
The push feature is taken from Justin Lebar's script available here:
https://github.com/jlebar/llvm-repo-tools/
Reviewers: jlebar
Subscribers: mgorny, modocache, llvm-commits
Differential Revision: https://reviews.llvm.org/D26334
Added:
llvm/trunk/utils/git-svn/git-llvm (with props)
Modified:
llvm/trunk/CMakeLists.txt
llvm/trunk/docs/GettingStarted.rst
Modified: llvm/trunk/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/CMakeLists.txt?rev=286123&r1=286122&r2=286123&view=diff
==============================================================================
--- llvm/trunk/CMakeLists.txt (original)
+++ llvm/trunk/CMakeLists.txt Mon Nov 7 11:40:28 2016
@@ -93,6 +93,25 @@ if(CMAKE_HOST_APPLE AND APPLE)
endif()
endif()
+# Git Mono-Repo handling: automatically set the LLVM_EXTERNAL_${project}_SOURCE_DIR
+set(LLVM_ALL_PROJECTS "clang;libcxx;libcxxabi;lldb;compiler-rt;lld;polly")
+set(LLVM_ENABLE_PROJECTS "" CACHE STRING
+ "Semicolon-separated list of projects to build (${LLVM_ALL_PROJECTS}), or \"all\".")
+if( LLVM_ENABLE_PROJECTS STREQUAL "all" )
+ set( LLVM_ENABLE_PROJECTS ${LLVM_ALL_PROJECTS})
+endif()
+foreach(proj ${LLVM_ENABLE_PROJECTS})
+ set(PROJ_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}")
+ if(NOT EXISTS "${PROJ_DIR}" OR NOT IS_DIRECTORY "${PROJ_DIR}")
+ message(FATAL_ERROR "LLVM_ENABLE_PROJECTS requests ${proj} but directory not found: ${PROJ_DIR}")
+ endif()
+ string(TOUPPER "${proj}" upper_proj)
+ set(LLVM_EXTERNAL_${upper_proj}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}")
+ if (proj STREQUAL "clang")
+ set(LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../clang-tools-extra")
+ endif()
+endforeach()
+
# The following only works with the Ninja generator in CMake >= 3.0.
set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
"Define the maximum number of concurrent compilation jobs.")
Modified: llvm/trunk/docs/GettingStarted.rst
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/docs/GettingStarted.rst?rev=286123&r1=286122&r2=286123&view=diff
==============================================================================
--- llvm/trunk/docs/GettingStarted.rst (original)
+++ llvm/trunk/docs/GettingStarted.rst Mon Nov 7 11:40:28 2016
@@ -680,6 +680,60 @@ about files with uncommitted changes. Th
Please, refer to the Git-SVN manual (``man git-svn``) for more information.
+For developers to work with a git monorepo
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. note::
+
+ This set-up is using unofficial mirror hosted on GitHub, use with caution.
+
+To set up a clone of all the llvm projects using a unified repository:
+
+.. code-block:: console
+
+ % export TOP_LEVEL_DIR=`pwd`
+ % git clone https://github.com/llvm-project/llvm-project/
+ % cd llvm-project
+ % git config branch.master.rebase true
+
+You can configure various build directory from this clone, starting with a build
+of LLVM alone:
+
+.. code-block:: console
+
+ % cd $TOP_LEVEL_DIR
+ % mkdir llvm-build && cd llvm-build
+ % cmake -GNinja ../llvm-project/llvm
+
+Or lldb:
+
+.. code-block:: console
+
+ % cd $TOP_LEVEL_DIR
+ % mkdir lldb-build && cd lldb-build
+ % cmake -GNinja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS=lldb
+
+Or a combination of multiple projects:
+
+.. code-block:: console
+
+ % cd $TOP_LEVEL_DIR
+ % mkdir clang-build && cd clang-build
+ % cmake -GNinja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS="clang;libcxx;compiler-rt"
+
+A helper script is provided in `llvm/utils/git-svn/git-llvm`. After you add it
+to your path, you can push committed changes upstream with `git llvm push`.
+
+.. code-block:: console
+
+ % export PATH=$PATH:$TOP_LEVEL_DIR/llvm-project/llvm/utils/git-svn/
+ % git llvm push
+
+While this is using SVN under the hood, it does not require any interaction from
+you with git-svn.
+After a few minutes, `git pull` should get back the changes as they were
+commited.
+
Local LLVM Configuration
------------------------
Added: llvm/trunk/utils/git-svn/git-llvm
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/git-svn/git-llvm?rev=286123&view=auto
==============================================================================
--- llvm/trunk/utils/git-svn/git-llvm (added)
+++ llvm/trunk/utils/git-svn/git-llvm Mon Nov 7 11:40:28 2016
@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+#
+# ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+# ==------------------------------------------------------------------------==#
+
+r"""
+git-llvm integration
+====================
+
+This file provides integration for git.
+
+For further details, run:
+
+ git llvm -h
+
+Requires Python 2.7
+"""
+
+from __future__ import print_function
+import argparse
+import collections
+import contextlib
+import errno
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+
+
+desc = '''
+TODO
+'''
+
+# It's *almost* a straightforward mapping from the monorepo to svn...
+GIT_TO_SVN_DIR = {
+ d: (d + '/trunk')
+ for d in [
+ 'clang-tools-extra',
+ 'compiler-rt',
+ 'dragonegg',
+ 'klee',
+ 'libclc',
+ 'libcxx',
+ 'libcxxabi',
+ 'lld',
+ 'lldb',
+ 'llvm',
+ 'polly',
+ ]
+}
+GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
+
+VERBOSE = False
+QUIET = False
+
+
+def eprint(*args, **kwargs):
+ print(*args, file=sys.stderr, **kwargs)
+
+
+def log(*args, **kwargs):
+ if QUIET:
+ return
+ print(*args, **kwargs)
+
+
+def log_verbose(*args, **kwargs):
+ if not VERBOSE:
+ return
+ print(*args, **kwargs)
+
+
+def die(msg):
+ eprint(msg)
+ sys.exit(1)
+
+
+def shell(cmd, strip=True, cwd=None, stdin=None):
+ log_verbose('Running: %s' % ' '.join(cmd))
+
+ start = time.time()
+ p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+ stdout, stderr = p.communicate(input=stdin)
+ elapsed = time.time() - start
+
+ if elapsed >= .1:
+ log_verbose('Warning: command took %0.1fs' % elapsed)
+
+ if p.returncode == 0:
+ if stderr:
+ eprint('`%s` printed to stderr:' % ' '.join(args))
+ eprint(stderr.rstrip())
+ if strip:
+ stdout = stdout.rstrip('\r\n')
+ return stdout
+ eprint('`%s` returned %s' % (' '.join(args), p.returncode))
+ if stderr:
+ eprint(stderr.rstrip())
+ sys.exit(2)
+
+
+def git(*cmd, **kwargs):
+ return shell(['git'] + list(cmd), kwargs.get('strip', True))
+
+
+def svn(cwd, *cmd, **kwargs):
+ # TODO: Better way to do default arg when we have *cmd?
+ return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None))
+
+
+def get_default_rev_range():
+ # Get the branch tracked by the current branch, as set by
+ # git branch --set-upstream-to See http://serverfault.com/a/352236/38694.
+ cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
+ upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
+ cur_branch)
+ if not upstream_branch:
+ upstream_branch = 'origin/master'
+
+ # Get the newest common ancestor between HEAD and our upstream branch.
+ upstream_rev = git('merge-base', 'HEAD', upstream_branch)
+ return '%s..' % upstream_rev
+
+
+def get_revs_to_push(rev_range):
+ if not rev_range:
+ rev_range = get_default_rev_range()
+ # Use git show rather than some plumbing command to figure out which revs
+ # are in rev_range because it handles single revs (HEAD^) and ranges
+ # (foo..bar) like we want.
+ revs = git('show', '--reverse', '--quiet',
+ '--pretty=%h', rev_range).split('\n')
+ # filter empty entries
+ revs = [r for r in revs if r != '']
+ if not revs:
+ die('Nothing to push: No revs in range %s.' % rev_range)
+ return revs
+
+
+def clean_and_update_svn(svn_repo):
+ svn(svn_repo, 'revert', '-R', '.')
+
+ # Unfortunately it appears there's no svn equivalent for git clean, so we
+ # have to do it ourselves.
+ for line in svn(svn_repo, 'status').split('\n'):
+ if not line.startswith('?'):
+ continue
+ sp = line.split(' ', 1)
+ if len(sp) != 2:
+ raise RuntimeError('Unexpected svn status line: %s' % line)
+ os.remove(os.path.join(svn_repo, sp[1].strip()))
+ svn(svn_repo, 'update')
+
+
+def svn_init(svn_root):
+ if not os.path.exists(svn_root):
+ print('Creating svn staging directory: (%s)' % (svn_root))
+ os.makedirs(svn_root)
+ print('This is a one-time initialization, please be patient for a few '
+ ' minutes...')
+ svn(svn_root, 'checkout', '--depth=immediates',
+ 'https://llvm.org/svn/llvm-project/', '.')
+ svn_dirs = list(GIT_TO_SVN_DIR.values())
+ svn(svn_root, 'update', *svn_dirs)
+ print('svn staging area ready in \'%s\'' % (svn_root))
+ if not os.path.isdir(svn_root):
+ die('Can\'t initialize svn staging dir (%s)' % (svn_root))
+
+
+def svn_push(args):
+ '''Push changes back to SVN: this is extracted from Justin Lebar's script
+ available here: https://github.com/jlebar/llvm-repo-tools/
+ '''
+ # Get the git root
+ git_root = git('rev-parse', '--show-toplevel')
+ if not os.path.isdir(git_root):
+ die('Can\'t find git root dir')
+
+ # Push from the root of the git repo
+ os.chdir(git_root)
+
+ # We need a staging area for SVN, let's hide it in the .git directory.
+ svn_root = os.path.join(git_root, '.git', 'llvm-upstream-svn')
+ svn_init(svn_root)
+
+ rev_range = args.rev_range
+ dry_run = args.dry_run
+ revs = get_revs_to_push(rev_range)
+ log('Pushing %d commit%s:\n%s' %
+ (len(revs), 's' if len(revs) != 1
+ else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c)
+ for c in revs)))
+ for r in revs:
+ clean_and_update_svn(svn_root)
+ push(svn_root, r, dry_run)
+
+
+def first_dirname(d):
+ while True:
+ (head, tail) = os.path.split(d)
+ if not head or head == '/':
+ return tail
+ d = head
+
+
+def push(svn_repo, rev, dry_run):
+ files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
+ rev).split('\n')
+ subrepos = {first_dirname(f) for f in files}
+ if not subrepos:
+ raise RuntimeError('Empty diff for rev %s?' % rev)
+
+ status = svn(svn_repo, 'status')
+ if status:
+ die('Can\'t push git rev %s because svn status is not empty:\n%s' %
+ (rev, status))
+
+ for sr in subrepos:
+ diff = git('show', '--binary', rev, '--', sr, strip=False)
+ svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
+ # git is the only thing that can handle its own patches...
+ log_verbose('Apply patch: %s' % diff)
+ shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff)
+
+ status_lines = svn(svn_repo, 'status').split('\n')
+
+ # Check that patch didn't dump any .orig or .rej files.
+ for line in status_lines:
+ if line.endswith('.rej') or line.endswith('.orig'):
+ raise RuntimeError('patch left a .orig/.rej file in the svn repo: '
+ '%s.' % line)
+ for l in (l for l in status_lines if l.startswith('?')):
+ svn(svn_repo, 'add', l[1:].strip())
+ for l in (l for l in status_lines if l.startswith('!')):
+ svn(svn_repo, 'remove', l[1:].strip())
+
+ # Now we're ready to commit.
+ commit_msg = git('show', '--pretty=%B', '--quiet', rev)
+ if not dry_run:
+ log(svn(svn_repo, 'commit', '-m', commit_msg))
+ log('Committed %s to svn.' % rev)
+ else:
+ log('Would have committed %s to svn, if this weren\'t a dry run.'
+ % rev)
+
+
+if __name__ == '__main__':
+ argv = sys.argv[1:]
+ p = argparse.ArgumentParser(
+ prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=desc)
+ subparsers = p.add_subparsers(title='subcommands',
+ description='valid subcommands',
+ help='additional help')
+ p.add_argument('-q', '--quiet', action='count', default=0,
+ help='print less information')
+ p.add_argument('-v', '--verbose', action='count', default=0,
+ help='print more information')
+
+ parser_push = subparsers.add_parser(
+ 'push', help='push changes back to the LLVM SVN repository')
+ parser_push.add_argument(
+ '-n',
+ '--dry-run',
+ dest='dry_run',
+ action='store_true',
+ help='Do everything other than commit to svn. Leaves junk in the svn '
+ 'repo, so probably will not work well if you try to commit more '
+ 'than one rev.')
+ parser_push.add_argument(
+ 'rev_range',
+ metavar='GIT_REVS',
+ type=str,
+ nargs='?',
+ help='revs to push (default: everything not in the branch\'s '
+ 'upstream, or not in origin/master if the branch lacks '
+ 'an explicit upstream)')
+ parser_push.set_defaults(func=svn_push)
+ args = p.parse_args(argv)
+ VERBOSE = args.verbose
+ QUIET = args.quiet
+
+ if QUIET and VERBOSE:
+ die('Error -v and -q cannot be used together: make your mind!')
+
+ # Dispatch to the right subcommand
+ args.func(args)
Propchange: llvm/trunk/utils/git-svn/git-llvm
------------------------------------------------------------------------------
svn:executable = *
More information about the llvm-commits
mailing list