Re: mercurial to git

[Theodore Tso <tytso@mit.edu>]

On Tue, Mar 06, 2007 at 06:08:02PM -0500, Josef Sipek wrote:
> I'll try to dig up the code, and put it up somewhere...

Here's a hacked up version of Stelian Pop's converter code that I used
for an initial test conversion of e2fsprogs from hg to git.  The main
improvements from Stelian's is that it's a bit faster by caching the
results of "hg log", and that it handles parses the Signed-off-by:
headers to feed in into the ChangeSet's Author identity (as distinct
from the committer identity, which it gets from the hg information).

The other change which I added was add a pretty kludgy committer name
cannonicalizer, since there the commiter information dates is pretty
grotty.  That's because the e2fsprogs source repository has over the
years been converted from CVS, to BitKeeper, to Mercurial, and now at
some point soon when I'm happy with a decent hg-to-git tool, to git.

My plan was to rewrite the converter to call Mercurial's python
classes directly (using the equivalent python code to 'hg manifest'
and 'hg cat' to speed things up enormously, compared to checking out
each revision one at a time and then using git to figure out which
files had been added/changed/deleted), and to interface it into
git-fast-import, and make the necessary changes (including more
intelligent handling of .hgtags) so that the conversion could be
bidrectional.

But if I can convince someone else to do the work, especially if their
converter handles the Signed-off-by: parsing, and making sure the
author and commit dates are properly set, that would certainly be a
bonus.  :-)

						- Ted

P.S.  Oh yes, my plan was to use Python's ConfigParser class to store
the author cannonicalization information, instead of hard-coding the
data into the python script.  Code snippets to do this available on
request; it was pretty trivial to do.

#! /usr/bin/python

""" hg-to-git.py - A Mercurial to GIT converter

    Copyright (C)2007 Stelian Pop <stelian@xxxxxxxxxx>

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2, or (at your option)
    any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
"""

import os, os.path, sys
import tempfile, popen2, pickle, getopt
import re

# Maps hg version -> git version
hgvers = {}
# List of children for each hg revision
hgchildren = {}
# Current branch for each hg revision
hgbranch = {}

#------------------------------------------------------------------------------

def usage():

        print """\
%s: [OPTIONS] <hgprj>

options:
    -s, --gitstate=FILE: name of the state to be saved/read
                         for incrementals

required:
    hgprj:  name of the HG project to import (directory)
""" % sys.argv[0]

#------------------------------------------------------------------------------

def getgitenv(user, author, date):
    env = ''
    if author == '':
	author = user

    elems = re.compile('(.*?)\s+<(.*)>').match(user)
    if elems:
        env += 'export GIT_COMMITER_NAME="%s" ;' % elems.group(1)
        env += 'export GIT_COMMITER_EMAIL="%s" ;' % elems.group(2)
    else:
        env += 'export GIT_COMMITER_NAME="%s" ;' % user
        env += 'export GIT_COMMITER_EMAIL= ;'

    elems = re.compile('(.*?)\s+<(.*)>').match(author)
    if elems:
        env += 'export GIT_AUTHOR_NAME="%s" ;' % elems.group(1)
        env += 'export GIT_AUTHOR_EMAIL="%s" ;' % elems.group(2)
    else:
        env += 'export GIT_AUTHOR_NAME="%s" ;' % author
        env += 'export GIT_AUTHOR_EMAIL= ;'

    env += 'export GIT_AUTHOR_DATE="%s" ;' % date
    env += 'export GIT_COMMITTER_DATE="%s" ;' % date
    return env

#------------------------------------------------------------------------------

state = ''

try:
    opts, args = getopt.getopt(sys.argv[1:], 's:t:', ['gitstate=', 'tempdir='])
    for o, a in opts:
        if o in ('-s', '--gitstate'):
            state = a
            state = os.path.abspath(state)

    if len(args) != 1:
        raise('params')
except:
    usage()
    sys.exit(1)

hgprj = args[0]
os.chdir(hgprj)

if state:
    if os.path.exists(state):
        print 'State does exist, reading'
        f = open(state, 'r')
        hgvers = pickle.load(f)
    else:
        print 'State does not exist, first run'

tip = os.popen('hg tip | head -1 | cut -f 2 -d :').read().strip()
print 'tip is', tip

# Calculate the branches
print 'analysing the branches...'
hgchildren["0"] = ()
hgbranch["0"] = "master"
for cset in range(1, int(tip) + 1):
    hgchildren[str(cset)] = ()
    prnts = os.popen('hg log -r %d | grep ^parent: | cut -f 2 -d :' % cset).readlines()
    if len(prnts) > 0:
        parent = prnts[0].strip()
    else:
        parent = str(cset - 1)
    hgchildren[parent] += ( str(cset), )
    if len(prnts) > 1:
        mparent = prnts[1].strip()
        hgchildren[mparent] += ( str(cset), )
    else:
        mparent = None

    if mparent:
        # For merge changesets, take either one, preferably the 'master' branch
        if hgbranch[mparent] == 'master':
            hgbranch[str(cset)] = 'master'
        else:
            hgbranch[str(cset)] = hgbranch[parent]
    else:
        # Normal changesets
        # For first children, take the parent branch, for the others create a new branch
        if hgchildren[parent][0] == str(cset):
            hgbranch[str(cset)] = hgbranch[parent]
        else:
            hgbranch[str(cset)] = "branch-" + str(cset)

if not hgvers.has_key("0"):
    print 'creating repository'
    os.system('git-init-db')

# loop through every hg changeset
for cset in range(int(tip) + 1):

    # incremental, already seen
    if hgvers.has_key(str(cset)):
        continue

    # get info
    prnts = os.popen('hg log -r %d | grep ^parent: | cut -f 2 -d :' % cset).readlines()
    if len(prnts) > 0:
        parent = prnts[0].strip()
    else:
        parent = str(cset - 1)
    if len(prnts) > 1:
        mparent = prnts[1].strip()
    else:
        mparent = None

    (fdlog, filelog) = tempfile.mkstemp()
    logtxt = os.popen('hg log -r %d -v' % cset).read().strip()
    os.write(fdlog, logtxt)
    os.close(fdlog)

    (fdcomment, filecomment) = tempfile.mkstemp()
    csetcomment = os.popen('grep -v ^changeset: < %s | grep -v ^parent: | grep -v ^user: | grep -v ^date | grep -v ^files: | grep -v ^description: | grep -v ^tag:' % filelog).read().strip()
    os.write(fdcomment, csetcomment)
    os.close(fdcomment)

    date = os.popen('grep -m 1 ^date: < %s | cut -f 2- -d :' % filelog).read().strip()

    tag = os.popen('grep -m 1 ^tag: < %s | cut -f 2- -d :' % filelog).read().strip()

    user = os.popen('grep -m 1 ^user: < %s | cut -f 2- -d :' % filelog).read().strip()
    if user == 'tytso@mit.edu':
	user = "Theodore Ts'o <tytso@mit.edu>"
    if user == 'tytso@think.thunk.org':
	user = "Theodore Ts'o <tytso@mit.edu>"
    if user == 'tytso@snap.thunk.org':
	user = "Theodore Ts'o <tytso@mit.edu>"
    if user == 'tytso@fs.thunk.org':
	user = "Theodore Ts'o <tytso@mit.edu>"
    if user == 'tytso@voltaire.debian.org':
	user = "Theodore Ts'o <tytso@mit.edu>"
    if user == 'tytso@who-could-of.thunk.org':
	user = "Theodore Ts'o <tytso@mit.edu>"
    if user == 'tytso@universal.(none)':
	user = "Theodore Ts'o <tytso@mit.edu>"
    if user == 'tytso@theodore-tsos-computer.local':
	user = "Theodore Ts'o <tytso@mit.edu>"

    if user == 'adilger@clusterfs.com':
	user = "Andreas Dilger <adilger@clusterfs.com>"
    if user == 'adilger@lynx.adilger.int':
	user = "Andreas Dilger <adilger@clusterfs.com>"
    if user == 'root@lynx.adilger.int':
	user = "Andreas Dilger <adilger@clusterfs.com>"
    if user == 'matthias.andree@gmx.de':
	user = "Matthias Andree <matthias.andree@gmx.de>"

    if user == 'laptop@duncow.home.oldelvet.org.uk':
	user = "Richard Mortimer <richm@oldelvet.org.uk>"

    if user == 'sct@redhat.com':
	user = 'Stephen Tweedie <sct@redhat.com>'
    if user == 'sct@sisko.scot.redhat.com':
	user = 'Stephen Tweedie <sct@redhat.com>'

    if user == 'paubert@gra-vd1.iram.es':
	user = 'Gabriel Paubert <paubert@iram.es>'

    author = os.popen('grep -m 1 ^Signed-off-by: < %s | cut -f 2- -d :' % filelog).read().strip()
    if author == '"Theodore Ts\'o" <tytso@mit.edu>':
	author = "Theodore Ts'o <tytso@mit.edu>"

    os.unlink(filelog)

    print '-----------------------------------------'
    print 'cset:', cset
    print 'branch:', hgbranch[str(cset)]
    print 'user:', user
    print 'author:', author
    print 'date:', date
    print 'comment:', csetcomment
    print 'parent:', parent
    if mparent:
        print 'mparent:', mparent
    if tag:
        print 'tag:', tag
    print '-----------------------------------------'

    # checkout the parent if necessary
    if cset != 0:
        if hgbranch[str(cset)] == "branch-" + str(cset):
            print 'creating new branch', hgbranch[str(cset)]
            os.system('git-checkout -b %s %s' % (hgbranch[str(cset)], hgvers[parent]))
        else:
            print 'checking out branch', hgbranch[str(cset)]
            os.system('git-checkout %s' % hgbranch[str(cset)])

    # merge
    if mparent:
        if hgbranch[parent] == hgbranch[str(cset)]:
            otherbranch = hgbranch[mparent]
        else:
            otherbranch = hgbranch[parent]
        print 'merging', otherbranch, 'into', hgbranch[str(cset)]
        os.system(getgitenv(user, author, date) + 'git-merge --no-commit -s ours "" %s %s' % (hgbranch[str(cset)], otherbranch))

    # remove everything except .git and .hg directories
    os.system('find . \( -path "./.hg" -o -path "./.git" \) -prune -o ! -name "." -print | xargs rm -rf')

    # repopulate with checkouted files
    os.system('hg update -C %d' % cset)

    # add new files
    os.system('git-ls-files -x .hg --others | git-update-index --add --stdin')
    # delete removed files
    os.system('git-ls-files -x .hg --deleted | git-update-index --remove --stdin')

    # commit
    os.system(getgitenv(user, author, date) + 'git-commit -a -F %s' % filecomment)
    os.unlink(filecomment)

    # tag
    if tag and tag != 'tip':
        os.system(getgitenv(user, author, date) + 'git-tag %s' % tag)

    # delete branch if not used anymore...
    if mparent and len(hgchildren[str(cset)]):
        print "Deleting unused branch:", otherbranch
        os.system('git-branch -d %s' % otherbranch)

    # retrieve and record the version
    vvv = os.popen('git-show | head -1').read()
    vvv = vvv[vvv.index(' ') + 1 : ].strip()
    print 'record', cset, '->', vvv
    hgvers[str(cset)] = vvv

os.system('git-repack -a -d')

# write the state for incrementals
if state:
    print 'Writing state'
    f = open(state, 'w')
    pickle.dump(hgvers, f)

# vim: et ts=8 sw=4 sts=4