Add some facilities to work with a git monorepo (experimental setup)

Summary:
Some changes are made to cmake, especially the addition of a new
LLVM_ENABLE_PROJECTS option that makes the build system aware of
the monorepo directory structure.

Also a new script is added in llvm/utils/git-svn/. When present in
the $PATH, it enables a `git llvm` command. It is providing at this
point only the ability to push from the git monorepo: `git llvm push`.
It is intended to evolves with more features, for instance I plan on
features like `git llvm show r284955` to help working with sequential
revision numbers.
The push feature is taken from Justin Lebar's script available here:
https://github.com/jlebar/llvm-repo-tools/

Reviewers: jlebar

Subscribers: mgorny, modocache, llvm-commits

Differential Revision: https://reviews.llvm.org/D26334

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@286123 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 296 insertions, 0 deletions

diff --git a/utils/git-svn/git-llvm b/utils/git-svn/git-llvm
new file mode 100755
index 00000000000..353fd128260
--- /dev/null
+++ b/utils/git-svn/git-llvm
@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+#
+# ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+# ==------------------------------------------------------------------------==#
+
+r"""
+git-llvm integration
+====================
+
+This file provides integration for git.
+
+For further details, run:
+
+    git llvm -h
+
+Requires Python 2.7
+"""
+
+from __future__ import print_function
+import argparse
+import collections
+import contextlib
+import errno
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+
+
+desc = '''
+TODO
+'''
+
+# It's *almost* a straightforward mapping from the monorepo to svn...
+GIT_TO_SVN_DIR = {
+    d: (d + '/trunk')
+    for d in [
+        'clang-tools-extra',
+        'compiler-rt',
+        'dragonegg',
+        'klee',
+        'libclc',
+        'libcxx',
+        'libcxxabi',
+        'lld',
+        'lldb',
+        'llvm',
+        'polly',
+    ]
+}
+GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
+
+VERBOSE = False
+QUIET = False
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def log(*args, **kwargs):
+    if QUIET:
+        return
+    print(*args, **kwargs)
+
+
+def log_verbose(*args, **kwargs):
+    if not VERBOSE:
+        return
+    print(*args, **kwargs)
+
+
+def die(msg):
+    eprint(msg)
+    sys.exit(1)
+
+
+def shell(cmd, strip=True, cwd=None, stdin=None):
+    log_verbose('Running: %s' % ' '.join(cmd))
+
+    start = time.time()
+    p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    stdout, stderr = p.communicate(input=stdin)
+    elapsed = time.time() - start
+
+    if elapsed >= .1:
+        log_verbose('Warning: command took %0.1fs' % elapsed)
+
+    if p.returncode == 0:
+        if stderr:
+            eprint('`%s` printed to stderr:' % ' '.join(args))
+            eprint(stderr.rstrip())
+        if strip:
+            stdout = stdout.rstrip('\r\n')
+        return stdout
+    eprint('`%s` returned %s' % (' '.join(args), p.returncode))
+    if stderr:
+        eprint(stderr.rstrip())
+    sys.exit(2)
+
+
+def git(*cmd, **kwargs):
+    return shell(['git'] + list(cmd), kwargs.get('strip', True))
+
+
+def svn(cwd, *cmd, **kwargs):
+    # TODO: Better way to do default arg when we have *cmd?
+    return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None))
+
+
+def get_default_rev_range():
+    # Get the branch tracked by the current branch, as set by
+    # git branch --set-upstream-to  See http://serverfault.com/a/352236/38694.
+    cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
+    upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
+                          cur_branch)
+    if not upstream_branch:
+        upstream_branch = 'origin/master'
+
+    # Get the newest common ancestor between HEAD and our upstream branch.
+    upstream_rev = git('merge-base', 'HEAD', upstream_branch)
+    return '%s..' % upstream_rev
+
+
+def get_revs_to_push(rev_range):
+    if not rev_range:
+        rev_range = get_default_rev_range()
+    # Use git show rather than some plumbing command to figure out which revs
+    # are in rev_range because it handles single revs (HEAD^) and ranges
+    # (foo..bar) like we want.
+    revs = git('show', '--reverse', '--quiet',
+               '--pretty=%h', rev_range).split('\n')
+    # filter empty entries
+    revs = [r for r in revs if r != '']
+    if not revs:
+        die('Nothing to push: No revs in range %s.' % rev_range)
+    return revs
+
+
+def clean_and_update_svn(svn_repo):
+    svn(svn_repo, 'revert', '-R', '.')
+
+    # Unfortunately it appears there's no svn equivalent for git clean, so we
+    # have to do it ourselves.
+    for line in svn(svn_repo, 'status').split('\n'):
+        if not line.startswith('?'):
+            continue
+        sp = line.split(' ', 1)
+        if len(sp) != 2:
+            raise RuntimeError('Unexpected svn status line: %s' % line)
+        os.remove(os.path.join(svn_repo, sp[1].strip()))
+    svn(svn_repo, 'update')
+
+
+def svn_init(svn_root):
+    if not os.path.exists(svn_root):
+        print('Creating svn staging directory: (%s)' % (svn_root))
+        os.makedirs(svn_root)
+        print('This is a one-time initialization, please be patient for a few '
+              ' minutes...')
+        svn(svn_root, 'checkout', '--depth=immediates',
+            'https://llvm.org/svn/llvm-project/', '.')
+        svn_dirs = list(GIT_TO_SVN_DIR.values())
+        svn(svn_root, 'update', *svn_dirs)
+        print('svn staging area ready in \'%s\'' % (svn_root))
+    if not os.path.isdir(svn_root):
+        die('Can\'t initialize svn staging dir (%s)' % (svn_root))
+
+
+def svn_push(args):
+    '''Push changes back to SVN: this is extracted from Justin Lebar's script
+    available here: https://github.com/jlebar/llvm-repo-tools/
+    '''
+    # Get the git root
+    git_root = git('rev-parse', '--show-toplevel')
+    if not os.path.isdir(git_root):
+        die('Can\'t find git root dir')
+
+    # Push from the root of the git repo
+    os.chdir(git_root)
+
+    # We need a staging area for SVN, let's hide it in the .git directory.
+    svn_root = os.path.join(git_root, '.git', 'llvm-upstream-svn')
+    svn_init(svn_root)
+
+    rev_range = args.rev_range
+    dry_run = args.dry_run
+    revs = get_revs_to_push(rev_range)
+    log('Pushing %d commit%s:\n%s' %
+        (len(revs), 's' if len(revs) != 1
+         else '', '\n'.join('  ' + git('show', '--oneline', '--quiet', c)
+                            for c in revs)))
+    for r in revs:
+        clean_and_update_svn(svn_root)
+        push(svn_root, r, dry_run)
+
+
+def first_dirname(d):
+    while True:
+        (head, tail) = os.path.split(d)
+        if not head or head == '/':
+            return tail
+        d = head
+
+
+def push(svn_repo, rev, dry_run):
+    files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
+                rev).split('\n')
+    subrepos = {first_dirname(f) for f in files}
+    if not subrepos:
+        raise RuntimeError('Empty diff for rev %s?' % rev)
+
+    status = svn(svn_repo, 'status')
+    if status:
+        die('Can\'t push git rev %s because svn status is not empty:\n%s' %
+            (rev, status))
+
+    for sr in subrepos:
+        diff = git('show', '--binary', rev, '--', sr, strip=False)
+        svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
+        # git is the only thing that can handle its own patches...
+        log_verbose('Apply patch: %s' % diff)
+        shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff)
+
+    status_lines = svn(svn_repo, 'status').split('\n')
+
+    # Check that patch didn't dump any .orig or .rej files.
+    for line in status_lines:
+        if line.endswith('.rej') or line.endswith('.orig'):
+            raise RuntimeError('patch left a .orig/.rej file in the svn repo: '
+                               '%s.' % line)
+    for l in (l for l in status_lines if l.startswith('?')):
+        svn(svn_repo, 'add', l[1:].strip())
+    for l in (l for l in status_lines if l.startswith('!')):
+        svn(svn_repo, 'remove', l[1:].strip())
+
+    # Now we're ready to commit.
+    commit_msg = git('show', '--pretty=%B', '--quiet', rev)
+    if not dry_run:
+        log(svn(svn_repo, 'commit', '-m', commit_msg))
+        log('Committed %s to svn.' % rev)
+    else:
+        log('Would have committed %s to svn, if this weren\'t a dry run.'
+            % rev)
+
+
+if __name__ == '__main__':
+    argv = sys.argv[1:]
+    p = argparse.ArgumentParser(
+        prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=desc)
+    subparsers = p.add_subparsers(title='subcommands',
+                                  description='valid subcommands',
+                                  help='additional help')
+    p.add_argument('-q', '--quiet', action='count', default=0,
+                   help='print less information')
+    p.add_argument('-v', '--verbose', action='count', default=0,
+                   help='print more information')
+
+    parser_push = subparsers.add_parser(
+        'push', help='push changes back to the LLVM SVN repository')
+    parser_push.add_argument(
+        '-n',
+        '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Do everything other than commit to svn.  Leaves junk in the svn '
+        'repo, so probably will not work well if you try to commit more '
+        'than one rev.')
+    parser_push.add_argument(
+        'rev_range',
+        metavar='GIT_REVS',
+        type=str,
+        nargs='?',
+        help='revs to push (default: everything not in the branch\'s '
+        'upstream, or not in origin/master if the branch lacks '
+        'an explicit upstream)')
+    parser_push.set_defaults(func=svn_push)
+    args = p.parse_args(argv)
+    VERBOSE = args.verbose
+    QUIET = args.quiet
+
+    if QUIET and VERBOSE:
+        die('Error -v and -q cannot be used together: make your mind!')
+
+    # Dispatch to the right subcommand
+    args.func(args)