[llvm] r347103 - Speed up git-llvm script by only svn up'ing affected directories.
James Y Knight via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 16 14:36:17 PST 2018
Author: jyknight
Date: Fri Nov 16 14:36:17 2018
New Revision: 347103
URL: http://llvm.org/viewvc/llvm-project?rev=347103&view=rev
Log:
Speed up git-llvm script by only svn up'ing affected directories.
Also, support modifications to toplevel files in git (which need to be
committed to "monorepo-root" in svn).
Differential Revision: https://reviews.llvm.org/D54341
Modified:
llvm/trunk/utils/git-svn/git-llvm
Modified: llvm/trunk/utils/git-svn/git-llvm
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/git-svn/git-llvm?rev=347103&r1=347102&r2=347103&view=diff
==============================================================================
--- llvm/trunk/utils/git-svn/git-llvm (original)
+++ llvm/trunk/utils/git-svn/git-llvm Fri Nov 16 14:36:17 2018
@@ -50,9 +50,11 @@ GIT_TO_SVN_DIR = {
'openmp',
'parallel-libs',
'polly',
+ 'pstl',
]
}
GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
+GIT_TO_SVN_DIR.update({'': 'monorepo-root/trunk'})
VERBOSE = False
QUIET = False
@@ -80,12 +82,12 @@ def die(msg):
sys.exit(1)
-def first_dirname(d):
- while True:
- (head, tail) = os.path.split(d)
- if not head or head == '/':
- return tail
- d = head
+def split_first_path_component(d):
+ # Assuming we have a git path, it'll use slashes even on windows...I hope.
+ if '/' in d:
+ return d.split('/', 1)
+ else:
+ return (d, None)
def get_dev_null():
@@ -98,7 +100,7 @@ def get_dev_null():
def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
ignore_errors=False, force_binary_stdin=False):
- log_verbose('Running: %s' % ' '.join(cmd))
+ log_verbose('Running in %s: %s' % (cwd, ' '.join(cmd)))
err_pipe = subprocess.PIPE
if ignore_errors:
@@ -127,6 +129,9 @@ def shell(cmd, strip=True, cwd=None, std
eprint(stderr.rstrip())
if strip:
stdout = stdout.rstrip('\r\n')
+ if VERBOSE:
+ for l in stdout.splitlines():
+ log_verbose("STDOUT: %s" % l)
return stdout
err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode)
eprint(err_msg)
@@ -181,7 +186,7 @@ def get_revs_to_push(rev_range):
return revs
-def clean_and_update_svn(svn_repo):
+def clean_svn(svn_repo):
svn(svn_repo, 'revert', '-R', '.')
# Unfortunately it appears there's no svn equivalent for git clean, so we
@@ -192,24 +197,19 @@ def clean_and_update_svn(svn_repo):
filename = line[1:].strip()
os.remove(os.path.join(svn_repo, filename))
- svn(svn_repo, 'update', *list(GIT_TO_SVN_DIR.values()))
-
def svn_init(svn_root):
if not os.path.exists(svn_root):
log('Creating svn staging directory: (%s)' % (svn_root))
os.makedirs(svn_root)
- log('This is a one-time initialization, please be patient for a few'
- ' minutes...')
svn(svn_root, 'checkout', '--depth=immediates',
'https://llvm.org/svn/llvm-project/', '.')
- svn(svn_root, 'update', *list(GIT_TO_SVN_DIR.values()))
log("svn staging area ready in '%s'" % svn_root)
if not os.path.isdir(svn_root):
die("Can't initialize svn staging dir (%s)" % svn_root)
-def fix_eol_style_native(rev, sr, svn_sr_path):
+def fix_eol_style_native(rev, svn_sr_path, files):
"""Fix line endings before applying patches with Unix endings
SVN on Windows will check out files with CRLF for files with the
@@ -219,9 +219,6 @@ def fix_eol_style_native(rev, sr, svn_sr
SVN will not commit a mass line ending re-doing because it detects the line
ending format for files with this property.
"""
- files = git('diff-tree', '--no-commit-id', '--name-only', '-r', rev, '--',
- sr).split('\n')
- files = [f.split('/', 1)[1] for f in files]
# Skip files that don't exist in SVN yet.
files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
# Use ignore_errors because 'svn propget' prints errors if the file doesn't
@@ -248,35 +245,80 @@ def fix_eol_style_native(rev, sr, svn_sr
if eol_style == 'native':
crlf_files.append(f)
if crlf_files:
- # Reformat all files with native SVN line endings to Unix format. SVN knows
- # files with native line endings are text files. It will commit just the
- # diff, and not a mass line ending change.
+ # Reformat all files with native SVN line endings to Unix format. SVN
+ # knows files with native line endings are text files. It will commit
+ # just the diff, and not a mass line ending change.
shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
+def get_all_parent_dirs(name):
+ parts = []
+ head, tail = os.path.split(name)
+ while head:
+ parts.append(head)
+ head, tail = os.path.split(head)
+ return parts
+
+def split_subrepo(f):
+ # Given a path, splits it into (subproject, rest-of-path). If the path is
+ # not in a subproject, returns ('', full-path).
+
+ subproject, remainder = split_first_path_component(f)
+
+ if subproject in GIT_TO_SVN_DIR:
+ return subproject, remainder
+ else:
+ return '', f
+
def svn_push_one_rev(svn_repo, rev, dry_run):
files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
rev).split('\n')
- subrepos = {first_dirname(f) for f in files}
- if not subrepos:
+ if not files:
raise RuntimeError('Empty diff for rev %s?' % rev)
+ # Split files by subrepo
+ subrepo_files = collections.defaultdict(list)
+ for f in files:
+ subrepo, remainder = split_subrepo(f)
+ subrepo_files[subrepo].append(remainder)
+
status = svn(svn_repo, 'status', '--no-ignore')
if status:
die("Can't push git rev %s because svn status is not empty:\n%s" %
(rev, status))
- for sr in subrepos:
+ svn_dirs_to_update = set()
+ for sr, files in subrepo_files.iteritems():
+ svn_sr_path = GIT_TO_SVN_DIR[sr]
+ for f in files:
+ svn_dirs_to_update.update(
+ get_all_parent_dirs(os.path.join(svn_sr_path, f)))
+
+ # Sort by length to ensure that the parent directories are passed to svn
+ # before child directories.
+ sorted_dirs_to_update = sorted(svn_dirs_to_update,
+ cmp=lambda x,y: cmp(len(x), len(y)))
+
+ # SVN update only in the affected directories.
+ svn(svn_repo, 'update', '--depth=immediates', *sorted_dirs_to_update)
+
+ for sr, files in subrepo_files.iteritems():
svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
if os.name == 'nt':
- fix_eol_style_native(rev, sr, svn_sr_path)
- diff = git('show', '--binary', rev, '--', sr, strip=False)
+ fix_eol_style_native(rev, svn_sr_path, files)
+ diff = git('show', '--binary', rev, '--',
+ *(os.path.join(sr, f) for f in files),
+ strip=False)
# git is the only thing that can handle its own patches...
log_verbose('Apply patch: %s' % diff)
+ if sr == '':
+ prefix_strip = '-p1'
+ else:
+ prefix_strip = '-p2'
try:
- # If we allow python to apply the diff in text mode, it will silently
- # convert \n to \r\n which git doesn't like.
- shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff,
- die_on_failure=False, force_binary_stdin=True)
+ # If we allow python to apply the diff in text mode, it will
+ # silently convert \n to \r\n which git doesn't like.
+ shell(['git', 'apply', prefix_strip, '-'], cwd=svn_sr_path,
+ stdin=diff, die_on_failure=False, force_binary_stdin=True)
except RuntimeError as e:
eprint("Patch doesn't apply: maybe you should try `git pull -r` "
"first?")
@@ -327,7 +369,7 @@ def cmd_push(args):
else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c)
for c in revs)))
for r in revs:
- clean_and_update_svn(svn_root)
+ clean_svn(svn_root)
svn_push_one_rev(svn_root, r, dry_run)
More information about the llvm-commits
mailing list