mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-11-25 20:23:11 +01:00
17ff4d329f
"svn update --depth=..." is, annoyingly, not a specification of the desired depth, but rather a _limit_ added on top of the "sticky" depth in the working-directory. However, if the directory doesn't exist yet, then it sets the sticky depth of the new directory entries. Unfortunately, the svn command-line has no way of expanding the depth of a directory from "empty" to "files", without also removing any already-expanded subdirectories. The way you're supposed to increase the depth of an existing directory is via --set-depth, but --set-depth=files will also remove any subdirs which were already requested. This change avoids getting into the state of ever needing to increase the depth of an existing directory from "empty" to "files" in the first place, by: 1. Use svn update --depth=files, not --depth=immediates. The latter has the effect of checking out the subdirectories and marking them as depth=empty. The former excludes sub-directories from the list of entries, which avoids the problem. 2. Explicitly populate missing parent directories. Using --parents seemed nice and easy, but it marks the parent dirs as depth=empty. Instead, check out parents explicitly if they're missing. llvm-svn: 347883
430 lines
14 KiB
Python
Executable File
430 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
|
|
#
|
|
# The LLVM Compiler Infrastructure
|
|
#
|
|
# This file is distributed under the University of Illinois Open Source
|
|
# License. See LICENSE.TXT for details.
|
|
#
|
|
# ==------------------------------------------------------------------------==#
|
|
|
|
"""
|
|
git-llvm integration
|
|
====================
|
|
|
|
This file provides integration for git.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
import argparse
|
|
import collections
|
|
import contextlib
|
|
import errno
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
assert sys.version_info >= (2, 7)
|
|
|
|
try:
|
|
dict.iteritems
|
|
except AttributeError:
|
|
# Python 3
|
|
def iteritems(d):
|
|
return iter(d.items())
|
|
else:
|
|
# Python 2
|
|
def iteritems(d):
|
|
return d.iteritems()
|
|
|
|
# It's *almost* a straightforward mapping from the monorepo to svn...
|
|
GIT_TO_SVN_DIR = {
|
|
d: (d + '/trunk')
|
|
for d in [
|
|
'clang-tools-extra',
|
|
'compiler-rt',
|
|
'debuginfo-tests',
|
|
'dragonegg',
|
|
'klee',
|
|
'libclc',
|
|
'libcxx',
|
|
'libcxxabi',
|
|
'libunwind',
|
|
'lld',
|
|
'lldb',
|
|
'llgo',
|
|
'llvm',
|
|
'openmp',
|
|
'parallel-libs',
|
|
'polly',
|
|
'pstl',
|
|
]
|
|
}
|
|
GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
|
|
GIT_TO_SVN_DIR.update({'': 'monorepo-root/trunk'})
|
|
|
|
VERBOSE = False
|
|
QUIET = False
|
|
dev_null_fd = None
|
|
|
|
|
|
def eprint(*args, **kwargs):
|
|
print(*args, file=sys.stderr, **kwargs)
|
|
|
|
|
|
def log(*args, **kwargs):
|
|
if QUIET:
|
|
return
|
|
print(*args, **kwargs)
|
|
|
|
|
|
def log_verbose(*args, **kwargs):
|
|
if not VERBOSE:
|
|
return
|
|
print(*args, **kwargs)
|
|
|
|
|
|
def die(msg):
|
|
eprint(msg)
|
|
sys.exit(1)
|
|
|
|
|
|
def split_first_path_component(d):
|
|
# Assuming we have a git path, it'll use slashes even on windows...I hope.
|
|
if '/' in d:
|
|
return d.split('/', 1)
|
|
else:
|
|
return (d, None)
|
|
|
|
|
|
def get_dev_null():
|
|
"""Lazily create a /dev/null fd for use in shell()"""
|
|
global dev_null_fd
|
|
if dev_null_fd is None:
|
|
dev_null_fd = open(os.devnull, 'w')
|
|
return dev_null_fd
|
|
|
|
|
|
def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
|
|
ignore_errors=False, text=True):
|
|
log_verbose('Running in %s: %s' % (cwd, ' '.join(cmd)))
|
|
|
|
err_pipe = subprocess.PIPE
|
|
if ignore_errors:
|
|
# Silence errors if requested.
|
|
err_pipe = get_dev_null()
|
|
|
|
start = time.time()
|
|
p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
|
|
stdin=subprocess.PIPE,
|
|
universal_newlines=text)
|
|
stdout, stderr = p.communicate(input=stdin)
|
|
elapsed = time.time() - start
|
|
|
|
log_verbose('Command took %0.1fs' % elapsed)
|
|
|
|
if p.returncode == 0 or ignore_errors:
|
|
if stderr and not ignore_errors:
|
|
eprint('`%s` printed to stderr:' % ' '.join(cmd))
|
|
eprint(stderr.rstrip())
|
|
if strip:
|
|
if text:
|
|
stdout = stdout.rstrip('\r\n')
|
|
else:
|
|
stdout = stdout.rstrip(b'\r\n')
|
|
if VERBOSE:
|
|
for l in stdout.splitlines():
|
|
log_verbose("STDOUT: %s" % l)
|
|
return stdout
|
|
err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode)
|
|
eprint(err_msg)
|
|
if stderr:
|
|
eprint(stderr.rstrip())
|
|
if die_on_failure:
|
|
sys.exit(2)
|
|
raise RuntimeError(err_msg)
|
|
|
|
|
|
def git(*cmd, **kwargs):
|
|
return shell(['git'] + list(cmd), **kwargs)
|
|
|
|
|
|
def svn(cwd, *cmd, **kwargs):
|
|
return shell(['svn'] + list(cmd), cwd=cwd, **kwargs)
|
|
|
|
def program_exists(cmd):
|
|
if sys.platform == 'win32' and not cmd.endswith('.exe'):
|
|
cmd += '.exe'
|
|
for path in os.environ["PATH"].split(os.pathsep):
|
|
if os.access(os.path.join(path, cmd), os.X_OK):
|
|
return True
|
|
return False
|
|
|
|
def get_default_rev_range():
|
|
# Get the branch tracked by the current branch, as set by
|
|
# git branch --set-upstream-to See http://serverfault.com/a/352236/38694.
|
|
cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
|
|
upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
|
|
cur_branch)
|
|
if not upstream_branch:
|
|
upstream_branch = 'origin/master'
|
|
|
|
# Get the newest common ancestor between HEAD and our upstream branch.
|
|
upstream_rev = git('merge-base', 'HEAD', upstream_branch)
|
|
return '%s..' % upstream_rev
|
|
|
|
|
|
def get_revs_to_push(rev_range):
|
|
if not rev_range:
|
|
rev_range = get_default_rev_range()
|
|
# Use git show rather than some plumbing command to figure out which revs
|
|
# are in rev_range because it handles single revs (HEAD^) and ranges
|
|
# (foo..bar) like we want.
|
|
revs = git('show', '--reverse', '--quiet',
|
|
'--pretty=%h', rev_range).splitlines()
|
|
if not revs:
|
|
die('Nothing to push: No revs in range %s.' % rev_range)
|
|
return revs
|
|
|
|
|
|
def clean_svn(svn_repo):
|
|
svn(svn_repo, 'revert', '-R', '.')
|
|
|
|
# Unfortunately it appears there's no svn equivalent for git clean, so we
|
|
# have to do it ourselves.
|
|
for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
|
|
if not line.startswith('?'):
|
|
continue
|
|
filename = line[1:].strip()
|
|
os.remove(os.path.join(svn_repo, filename))
|
|
|
|
|
|
def svn_init(svn_root):
|
|
if not os.path.exists(svn_root):
|
|
log('Creating svn staging directory: (%s)' % (svn_root))
|
|
os.makedirs(svn_root)
|
|
svn(svn_root, 'checkout', '--depth=empty',
|
|
'https://llvm.org/svn/llvm-project/', '.')
|
|
log("svn staging area ready in '%s'" % svn_root)
|
|
if not os.path.isdir(svn_root):
|
|
die("Can't initialize svn staging dir (%s)" % svn_root)
|
|
|
|
|
|
def fix_eol_style_native(rev, svn_sr_path, files):
|
|
"""Fix line endings before applying patches with Unix endings
|
|
|
|
SVN on Windows will check out files with CRLF for files with the
|
|
svn:eol-style property set to "native". This breaks `git apply`, which
|
|
typically works with Unix-line ending patches. Work around the problem here
|
|
by doing a dos2unix up front for files with svn:eol-style set to "native".
|
|
SVN will not commit a mass line ending re-doing because it detects the line
|
|
ending format for files with this property.
|
|
"""
|
|
# Skip files that don't exist in SVN yet.
|
|
files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
|
|
# Use ignore_errors because 'svn propget' prints errors if the file doesn't
|
|
# have the named property. There doesn't seem to be a way to suppress that.
|
|
eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
|
|
ignore_errors=True)
|
|
crlf_files = []
|
|
if len(files) == 1:
|
|
# No need to split propget output on ' - ' when we have one file.
|
|
if eol_props.strip() in ['native', 'CRLF']:
|
|
crlf_files = files
|
|
else:
|
|
for eol_prop in eol_props.split('\n'):
|
|
# Remove spare CR.
|
|
eol_prop = eol_prop.strip('\r')
|
|
if not eol_prop:
|
|
continue
|
|
prop_parts = eol_prop.rsplit(' - ', 1)
|
|
if len(prop_parts) != 2:
|
|
eprint("unable to parse svn propget line:")
|
|
eprint(eol_prop)
|
|
continue
|
|
(f, eol_style) = prop_parts
|
|
if eol_style == 'native':
|
|
crlf_files.append(f)
|
|
if crlf_files:
|
|
# Reformat all files with native SVN line endings to Unix format. SVN
|
|
# knows files with native line endings are text files. It will commit
|
|
# just the diff, and not a mass line ending change.
|
|
shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
|
|
|
|
def split_subrepo(f):
|
|
# Given a path, splits it into (subproject, rest-of-path). If the path is
|
|
# not in a subproject, returns ('', full-path).
|
|
|
|
subproject, remainder = split_first_path_component(f)
|
|
|
|
if subproject in GIT_TO_SVN_DIR:
|
|
return subproject, remainder
|
|
else:
|
|
return '', f
|
|
|
|
def get_all_parent_dirs(name):
|
|
parts = []
|
|
head, tail = os.path.split(name)
|
|
while head:
|
|
parts.append(head)
|
|
head, tail = os.path.split(head)
|
|
return parts
|
|
|
|
def svn_push_one_rev(svn_repo, rev, dry_run):
|
|
files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
|
|
rev).split('\n')
|
|
if not files:
|
|
raise RuntimeError('Empty diff for rev %s?' % rev)
|
|
|
|
# Split files by subrepo
|
|
subrepo_files = collections.defaultdict(list)
|
|
for f in files:
|
|
subrepo, remainder = split_subrepo(f)
|
|
subrepo_files[subrepo].append(remainder)
|
|
|
|
status = svn(svn_repo, 'status', '--no-ignore')
|
|
if status:
|
|
die("Can't push git rev %s because svn status is not empty:\n%s" %
|
|
(rev, status))
|
|
|
|
svn_dirs_to_update = set()
|
|
for sr, files in iteritems(subrepo_files):
|
|
svn_sr_path = GIT_TO_SVN_DIR[sr]
|
|
for f in files:
|
|
svn_dirs_to_update.add(
|
|
os.path.dirname(os.path.join(svn_sr_path, f)))
|
|
|
|
# We also need to svn update any parent directories which are not yet present
|
|
parent_dirs = set()
|
|
for dir in svn_dirs_to_update:
|
|
parent_dirs.update(get_all_parent_dirs(dir))
|
|
parent_dirs = set(dir for dir in parent_dirs
|
|
if not os.path.exists(os.path.join(svn_repo, dir)))
|
|
svn_dirs_to_update.update(parent_dirs)
|
|
|
|
# Sort by length to ensure that the parent directories are passed to svn
|
|
# before child directories.
|
|
sorted_dirs_to_update = sorted(svn_dirs_to_update, key=len)
|
|
|
|
# SVN update only in the affected directories.
|
|
svn(svn_repo, 'update', '--depth=files', *sorted_dirs_to_update)
|
|
|
|
for sr, files in iteritems(subrepo_files):
|
|
svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
|
|
if os.name == 'nt':
|
|
fix_eol_style_native(rev, svn_sr_path, files)
|
|
# We use text=False (and pass '--binary') so that we can get an exact
|
|
# diff that can be passed as-is to 'git apply' without any line ending,
|
|
# encoding, or other mangling.
|
|
diff = git('show', '--binary', rev, '--',
|
|
*(os.path.join(sr, f) for f in files),
|
|
strip=False, text=False)
|
|
# git is the only thing that can handle its own patches...
|
|
if sr == '':
|
|
prefix_strip = '-p1'
|
|
else:
|
|
prefix_strip = '-p2'
|
|
try:
|
|
shell(['git', 'apply', prefix_strip, '-'], cwd=svn_sr_path,
|
|
stdin=diff, die_on_failure=False, text=False)
|
|
except RuntimeError as e:
|
|
eprint("Patch doesn't apply: maybe you should try `git pull -r` "
|
|
"first?")
|
|
sys.exit(2)
|
|
|
|
status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
|
|
|
|
for l in (l for l in status_lines if (l.startswith('?') or
|
|
l.startswith('I'))):
|
|
svn(svn_repo, 'add', '--no-ignore', l[1:].strip())
|
|
for l in (l for l in status_lines if l.startswith('!')):
|
|
svn(svn_repo, 'remove', l[1:].strip())
|
|
|
|
# Now we're ready to commit.
|
|
commit_msg = git('show', '--pretty=%B', '--quiet', rev)
|
|
if not dry_run:
|
|
log(svn(svn_repo, 'commit', '-m', commit_msg, '--force-interactive'))
|
|
log('Committed %s to svn.' % rev)
|
|
else:
|
|
log("Would have committed %s to svn, if this weren't a dry run." % rev)
|
|
|
|
|
|
def cmd_push(args):
|
|
'''Push changes back to SVN: this is extracted from Justin Lebar's script
|
|
available here: https://github.com/jlebar/llvm-repo-tools/
|
|
|
|
Note: a current limitation is that git does not track file rename, so they
|
|
will show up in SVN as delete+add.
|
|
'''
|
|
# Get the git root
|
|
git_root = git('rev-parse', '--show-toplevel')
|
|
if not os.path.isdir(git_root):
|
|
die("Can't find git root dir")
|
|
|
|
# Push from the root of the git repo
|
|
os.chdir(git_root)
|
|
|
|
# We need a staging area for SVN, let's hide it in the .git directory.
|
|
dot_git_dir = git('rev-parse', '--git-common-dir')
|
|
svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn')
|
|
svn_init(svn_root)
|
|
|
|
rev_range = args.rev_range
|
|
dry_run = args.dry_run
|
|
revs = get_revs_to_push(rev_range)
|
|
log('Pushing %d commit%s:\n%s' %
|
|
(len(revs), 's' if len(revs) != 1
|
|
else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c)
|
|
for c in revs)))
|
|
for r in revs:
|
|
clean_svn(svn_root)
|
|
svn_push_one_rev(svn_root, r, dry_run)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if not program_exists('svn'):
|
|
die('error: git-llvm needs svn command, but svn is not installed.')
|
|
|
|
argv = sys.argv[1:]
|
|
p = argparse.ArgumentParser(
|
|
prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
description=__doc__)
|
|
subcommands = p.add_subparsers(title='subcommands',
|
|
description='valid subcommands',
|
|
help='additional help')
|
|
verbosity_group = p.add_mutually_exclusive_group()
|
|
verbosity_group.add_argument('-q', '--quiet', action='store_true',
|
|
help='print less information')
|
|
verbosity_group.add_argument('-v', '--verbose', action='store_true',
|
|
help='print more information')
|
|
|
|
parser_push = subcommands.add_parser(
|
|
'push', description=cmd_push.__doc__,
|
|
help='push changes back to the LLVM SVN repository')
|
|
parser_push.add_argument(
|
|
'-n',
|
|
'--dry-run',
|
|
dest='dry_run',
|
|
action='store_true',
|
|
help='Do everything other than commit to svn. Leaves junk in the svn '
|
|
'repo, so probably will not work well if you try to commit more '
|
|
'than one rev.')
|
|
parser_push.add_argument(
|
|
'rev_range',
|
|
metavar='GIT_REVS',
|
|
type=str,
|
|
nargs='?',
|
|
help="revs to push (default: everything not in the branch's "
|
|
'upstream, or not in origin/master if the branch lacks '
|
|
'an explicit upstream)')
|
|
parser_push.set_defaults(func=cmd_push)
|
|
args = p.parse_args(argv)
|
|
VERBOSE = args.verbose
|
|
QUIET = args.quiet
|
|
|
|
# Dispatch to the right subcommand
|
|
args.func(args)
|