changeset 5021:7628612b822c

Merge with bos

--- a/.hgignore	Thu Jul 26 14:02:49 2007 -0700
+++ b/.hgignore	Thu Jul 26 14:04:48 2007 -0700
@@ -4,6 +4,7 @@
 *.orig
 *.rej
 *~
+*.mergebackup
 *.o
 *.so
 *.pyc

--- a/hgext/convert/__init__.py	Thu Jul 26 14:02:49 2007 -0700
+++ b/hgext/convert/__init__.py	Thu Jul 26 14:04:48 2007 -0700
@@ -8,23 +8,23 @@
 from common import NoRepo, converter_source, converter_sink
 from cvs import convert_cvs
 from git import convert_git
-from hg import convert_mercurial
+from hg import mercurial_source, mercurial_sink
 from subversion import convert_svn
 
-import os, shutil
+import os, shlex, shutil
 from mercurial import hg, ui, util, commands
+from mercurial.i18n import _
 
 commands.norepo += " convert"
 
-converters = [convert_cvs, convert_git, convert_svn, convert_mercurial]
+converters = [convert_cvs, convert_git, convert_svn, mercurial_source,
+              mercurial_sink]
 
 def convertsource(ui, path, **opts):
     for c in converters:
-        if not hasattr(c, 'getcommit'):
-            continue
         try:
-            return c(ui, path, **opts)
-        except NoRepo:
+            return c.getcommit and c(ui, path, **opts)
+        except (AttributeError, NoRepo):
             pass
     raise util.Abort('%s: unknown repository type' % path)
 
@@ -32,34 +32,33 @@
     if not os.path.isdir(path):
         raise util.Abort("%s: not a directory" % path)
     for c in converters:
-        if not hasattr(c, 'putcommit'):
-            continue
         try:
-            return c(ui, path)
-        except NoRepo:
+            return c.putcommit and c(ui, path)
+        except (AttributeError, NoRepo):
             pass
     raise util.Abort('%s: unknown repository type' % path)
 
 class convert(object):
-    def __init__(self, ui, source, dest, mapfile, opts):
+    def __init__(self, ui, source, dest, revmapfile, filemapper, opts):
 
         self.source = source
         self.dest = dest
         self.ui = ui
         self.opts = opts
         self.commitcache = {}
-        self.mapfile = mapfile
-        self.mapfilefd = None
+        self.revmapfile = revmapfile
+        self.revmapfilefd = None
         self.authors = {}
         self.authorfile = None
+        self.mapfile = filemapper
 
         self.map = {}
         try:
-            origmapfile = open(self.mapfile, 'r')
-            for l in origmapfile:
+            origrevmapfile = open(self.revmapfile, 'r')
+            for l in origrevmapfile:
                 sv, dv = l[:-1].split()
                 self.map[sv] = dv
-            origmapfile.close()
+            origrevmapfile.close()
         except IOError:
             pass
 
@@ -151,14 +150,14 @@
         return s
 
     def mapentry(self, src, dst):
-        if self.mapfilefd is None:
+        if self.revmapfilefd is None:
             try:
-                self.mapfilefd = open(self.mapfile, "a")
+                self.revmapfilefd = open(self.revmapfile, "a")
             except IOError, (errno, strerror):
-                raise util.Abort("Could not open map file %s: %s, %s\n" % (self.mapfile, errno, strerror))
+                raise util.Abort("Could not open map file %s: %s, %s\n" % (self.revmapfile, errno, strerror))
         self.map[src] = dst
-        self.mapfilefd.write("%s %s\n" % (src, dst))
-        self.mapfilefd.flush()
+        self.revmapfilefd.write("%s %s\n" % (src, dst))
+        self.revmapfilefd.flush()
 
     def writeauthormap(self):
         authorfile = self.authorfile
@@ -190,32 +189,36 @@
         afile.close()
 
     def copy(self, rev):
-        c = self.commitcache[rev]
-        files = self.source.getchanges(rev)
+        commit = self.commitcache[rev]
+        do_copies = hasattr(self.dest, 'copyfile')
+        filenames = []
 
-        do_copies = (hasattr(c, 'copies') and hasattr(self.dest, 'copyfile'))
-
-        for f, v in files:
+        for f, v in self.source.getchanges(rev):
+            newf = self.mapfile(f)
+            if not newf:
+                continue
+            filenames.append(newf)
             try:
                 data = self.source.getfile(f, v)
             except IOError, inst:
-                self.dest.delfile(f)
+                self.dest.delfile(newf)
             else:
                 e = self.source.getmode(f, v)
-                self.dest.putfile(f, e, data)
+                self.dest.putfile(newf, e, data)
                 if do_copies:
-                    if f in c.copies:
-                        # Merely marks that a copy happened.
-                        self.dest.copyfile(c.copies[f], f)
+                    if f in commit.copies:
+                        copyf = self.mapfile(commit.copies[f])
+                        if copyf:
+                            # Merely marks that a copy happened.
+                            self.dest.copyfile(copyf, newf)
 
-
-        r = [self.map[v] for v in c.parents]
-        f = [f for f, v in files]
-        newnode = self.dest.putcommit(f, r, c)
+        parents = [self.map[r] for r in commit.parents]
+        newnode = self.dest.putcommit(filenames, parents, commit)
         self.mapentry(rev, newnode)
 
     def convert(self):
         try:
+            self.dest.before()
             self.source.setrevmap(self.map)
             self.ui.status("scanning source...\n")
             heads = self.source.getheads()
@@ -256,10 +259,92 @@
             self.cleanup()
 
     def cleanup(self):
-       if self.mapfilefd:
-           self.mapfilefd.close()
+        self.dest.after()
+        if self.revmapfilefd:
+            self.revmapfilefd.close()
+
+def rpairs(name):
+    e = len(name)
+    while e != -1:
+        yield name[:e], name[e+1:]
+        e = name.rfind('/', 0, e)
+
+class filemapper(object):
+    '''Map and filter filenames when importing.
+    A name can be mapped to itself, a new name, or None (omit from new
+    repository).'''
+
+    def __init__(self, ui, path=None):
+        self.ui = ui
+        self.include = {}
+        self.exclude = {}
+        self.rename = {}
+        if path:
+            if self.parse(path):
+                raise util.Abort(_('errors in filemap'))
 
-def _convert(ui, src, dest=None, mapfile=None, **opts):
+    def parse(self, path):
+        errs = 0
+        def check(name, mapping, listname):
+            if name in mapping:
+                self.ui.warn(_('%s:%d: %r already in %s list\n') %
+                             (lex.infile, lex.lineno, name, listname))
+                return 1
+            return 0
+        lex = shlex.shlex(open(path), path, True)
+        lex.wordchars += '!@#$%^&*()-=+[]{}|;:,./<>?'
+        cmd = lex.get_token()
+        while cmd:
+            if cmd == 'include':
+                name = lex.get_token()
+                errs += check(name, self.exclude, 'exclude')
+                self.include[name] = name
+            elif cmd == 'exclude':
+                name = lex.get_token()
+                errs += check(name, self.include, 'include')
+                errs += check(name, self.rename, 'rename')
+                self.exclude[name] = name
+            elif cmd == 'rename':
+                src = lex.get_token()
+                dest = lex.get_token()
+                errs += check(src, self.exclude, 'exclude')
+                self.rename[src] = dest
+            elif cmd == 'source':
+                errs += self.parse(lex.get_token())
+            else:
+                self.ui.warn(_('%s:%d: unknown directive %r\n') %
+                             (lex.infile, lex.lineno, cmd))
+                errs += 1
+            cmd = lex.get_token()
+        return errs
+
+    def lookup(self, name, mapping):
+        for pre, suf in rpairs(name):
+            try:
+                return mapping[pre], pre, suf
+            except KeyError, err:
+                pass
+        return '', name, ''
+        
+    def __call__(self, name):
+        if self.include:
+            inc = self.lookup(name, self.include)[0]
+        else:
+            inc = name
+        if self.exclude:
+            exc = self.lookup(name, self.exclude)[0]
+        else:
+            exc = ''
+        if not inc or exc:
+            return None
+        newpre, pre, suf = self.lookup(name, self.rename)
+        if newpre:
+            if suf:
+                return newpre + '/' + suf
+            return newpre
+        return name
+
+def _convert(ui, src, dest=None, revmapfile=None, **opts):
     """Convert a foreign SCM repository to a Mercurial one.
 
     Accepted source formats:
@@ -278,8 +363,8 @@
     basename of the source with '-hg' appended.  If the destination
     repository doesn't exist, it will be created.
 
-    If <mapfile> isn't given, it will be put in a default location
-    (<dest>/.hg/shamap by default).  The <mapfile> is a simple text
+    If <revmapfile> isn't given, it will be put in a default location
+    (<dest>/.hg/shamap by default).  The <revmapfile> is a simple text
     file that maps each source commit ID to the destination ID for
     that revision, like so:
     <source ID> <destination ID>
@@ -334,19 +419,22 @@
             shutil.rmtree(dest, True)
         raise
 
-    if not mapfile:
+    if not revmapfile:
         try:
-            mapfile = destc.mapfile()
+            revmapfile = destc.revmapfile()
         except:
-            mapfile = os.path.join(destc, "map")
+            revmapfile = os.path.join(destc, "map")
+
 
-    c = convert(ui, srcc, destc, mapfile, opts)
+    c = convert(ui, srcc, destc, revmapfile, filemapper(ui, opts['filemap']),
+                opts)
     c.convert()
 
 cmdtable = {
     "convert":
         (_convert,
          [('A', 'authors', '', 'username mapping filename'),
+          ('', 'filemap', '', 'remap file names using contents of file'),
           ('r', 'rev', '', 'import up to target revision REV'),
           ('', 'datesort', None, 'try to sort changesets by date')],
          'hg convert [OPTION]... SOURCE [DEST [MAPFILE]]'),

--- a/hgext/convert/common.py	Thu Jul 26 14:02:49 2007 -0700
+++ b/hgext/convert/common.py	Thu Jul 26 14:04:48 2007 -0700
@@ -3,16 +3,20 @@
 class NoRepo(Exception): pass
 
 class commit(object):
-    def __init__(self, **parts):
+    def __init__(self, author, date, desc, parents, branch=None, rev=None,
+                 copies={}):
         self.rev = None
         self.branch = None
-
-        for x in "author date desc parents".split():
-            if not x in parts:
-                raise util.Abort("commit missing field %s" % x)
-        self.__dict__.update(parts)
-        if not self.desc or self.desc.isspace():
+        self.author = author
+        self.date = date
+        if desc and not desc.isspace():
+            self.desc = desc
+        else:
             self.desc = '*** empty log message ***'
+        self.parents = parents
+        self.branch = branch
+        self.rev = rev
+        self.copies = copies
 
 class converter_source(object):
     """Conversion source interface"""
@@ -81,7 +85,7 @@
         """Return a list of this repository's heads"""
         raise NotImplementedError()
 
-    def mapfile(self):
+    def revmapfile(self):
         """Path to a file that will contain lines
         source_rev_id sink_rev_id
         mapping equivalent revision identifiers for each system."""

--- a/hgext/convert/hg.py	Thu Jul 26 14:02:49 2007 -0700
+++ b/hgext/convert/hg.py	Thu Jul 26 14:04:48 2007 -0700
@@ -1,20 +1,39 @@
 # hg backend for convert extension
 
+# Note for hg->hg conversion: Old versions of Mercurial didn't trim
+# the whitespace from the ends of commit messages, but new versions
+# do.  Changesets created by those older versions, then converted, may
+# thus have different hashes for changesets that are otherwise
+# identical.
+
+
 import os, time
-from mercurial import hg
+from mercurial.i18n import _
+from mercurial.node import *
+from mercurial import hg, lock, revlog, util
 
-from common import NoRepo, converter_sink
+from common import NoRepo, commit, converter_source, converter_sink
 
-class convert_mercurial(converter_sink):
+class mercurial_sink(converter_sink):
     def __init__(self, ui, path):
         self.path = path
         self.ui = ui
         try:
             self.repo = hg.repository(self.ui, path)
         except:
-            raise NoRepo("could open hg repo %s" % path)
+            raise NoRepo("could not open hg repo %s as sink" % path)
+        self.lock = None
+        self.wlock = None
 
-    def mapfile(self):
+    def before(self):
+        self.lock = self.repo.lock()
+        self.wlock = self.repo.wlock()
+
+    def after(self):
+        self.lock = None
+        self.wlock = None
+
+    def revmapfile(self):
         return os.path.join(self.path, ".hg", "shamap")
 
     def authorfile(self):
@@ -22,7 +41,7 @@
 
     def getheads(self):
         h = self.repo.changelog.heads()
-        return [ hg.hex(x) for x in h ]
+        return [ hex(x) for x in h ]
 
     def putfile(self, f, e, data):
         self.repo.wwrite(f, data, e)
@@ -40,7 +59,10 @@
             pass
 
     def putcommit(self, files, parents, commit):
-        seen = {}
+        if not files:
+            return hex(self.repo.changelog.tip())
+
+        seen = {hex(nullid): 1}
         pl = []
         for p in parents:
             if p not in seen:
@@ -63,7 +85,8 @@
             p1 = p2
             p2 = parents.pop(0)
             a = self.repo.rawcommit(files, text, commit.author, commit.date,
-                                    hg.bin(p1), hg.bin(p2), extra=extra)
+                                    bin(p1), bin(p2), extra=extra)
+            self.repo.dirstate.invalidate()
             text = "(octopus merge fixup)\n"
             p2 = hg.hex(self.repo.changelog.tip())
 
@@ -93,5 +116,59 @@
             if not oldlines: self.repo.add([".hgtags"])
             date = "%s 0" % int(time.mktime(time.gmtime()))
             self.repo.rawcommit([".hgtags"], "update tags", "convert-repo",
-                                date, self.repo.changelog.tip(), hg.nullid)
-            return hg.hex(self.repo.changelog.tip())
+                                date, self.repo.changelog.tip(), nullid)
+            return hex(self.repo.changelog.tip())
+
+class mercurial_source(converter_source):
+    def __init__(self, ui, path, rev=None):
+        converter_source.__init__(self, ui, path, rev)
+        self.repo = hg.repository(self.ui, path)
+        self.lastrev = None
+        self.lastctx = None
+
+    def changectx(self, rev):
+        if self.lastrev != rev:
+            self.lastctx = self.repo.changectx(rev)
+            self.lastrev = rev
+        return self.lastctx
+
+    def getheads(self):
+        return [hex(node) for node in self.repo.heads()]
+
+    def getfile(self, name, rev):
+        try:
+            return self.changectx(rev).filectx(name).data()
+        except revlog.LookupError, err:
+            raise IOError(err)
+
+    def getmode(self, name, rev):
+        m = self.changectx(rev).manifest()
+        return (m.execf(name) and 'x' or '') + (m.linkf(name) and 'l' or '')
+
+    def getchanges(self, rev):
+        ctx = self.changectx(rev)
+        m, a, r = self.repo.status(ctx.parents()[0].node(), ctx.node())[:3]
+        changes = [(name, rev) for name in m + a + r]
+        changes.sort()
+        return changes
+
+    def getcopies(self, ctx):
+        added = self.repo.status(ctx.parents()[0].node(), ctx.node())[1]
+        copies = {}
+        for name in added:
+            try:
+                copies[name] = ctx.filectx(name).renamed()[0]
+            except TypeError:
+                pass
+        return copies
+        
+    def getcommit(self, rev):
+        ctx = self.changectx(rev)
+        parents = [hex(p.node()) for p in ctx.parents() if p.node() != nullid]
+        return commit(author=ctx.user(), date=util.datestr(ctx.date()),
+                      desc=ctx.description(), parents=parents,
+                      branch=ctx.branch(), copies=self.getcopies(ctx))
+
+    def gettags(self):
+        tags = [t for t in self.repo.tagslist() if t[0] != 'tip']
+        return dict([(name, hex(node)) for name, node in tags])

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-convert	Thu Jul 26 14:04:48 2007 -0700
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+echo "[extensions]" >> $HGRCPATH
+echo "convert=" >> $HGRCPATH
+
+hg init a
+cd a
+echo a > a
+hg ci -d'0 0' -Ama
+hg cp a b
+hg ci -d'1 0' -mb
+hg rm a
+hg ci -d'2 0' -mc
+hg mv b a
+hg ci -d'3 0' -md
+echo a >> a
+hg ci -d'4 0' -me
+
+cd ..
+hg convert a
+hg --cwd a-hg pull ../a

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-convert.out	Thu Jul 26 14:04:48 2007 -0700
@@ -0,0 +1,14 @@
+adding a
+assuming destination a-hg
+initializing destination a-hg repository
+scanning source...
+sorting...
+converting...
+4 a
+3 b
+2 c
+1 d
+0 e
+pulling from ../a
+searching for changes
+no changes found