convert: replace fork with subprocess call.
--- a/hgext/convert/__init__.py Thu Aug 02 23:38:32 2007 +0200
+++ b/hgext/convert/__init__.py Mon Aug 06 21:49:26 2007 +0200
@@ -5,17 +5,17 @@
# This software may be used and distributed according to the terms
# of the GNU General Public License, incorporated herein by reference.
-from common import NoRepo, converter_source, converter_sink
+from common import NoRepo, converter_source, converter_sink, decodeargs
from cvs import convert_cvs
from git import convert_git
from hg import mercurial_source, mercurial_sink
from subversion import convert_svn
-import os, shlex, shutil
+import os, shlex, shutil, sys
from mercurial import hg, ui, util, commands
from mercurial.i18n import _
-commands.norepo += " convert"
+commands.norepo += " convert debug-svn-log"
converters = [convert_cvs, convert_git, convert_svn, mercurial_source,
mercurial_sink]
@@ -431,6 +431,15 @@
opts)
c.convert()
+def debugsvnlog(ui, **opts):
+ """Fetch SVN log in a subprocess and channel them back to parent to
+ avoid memory collection issues.
+ """
+ util.set_binary(sys.stdin)
+ util.set_binary(sys.stdout)
+ args = decodeargs(sys.stdin.read())
+ subversion.get_log_child(sys.stdout, *args)
+
cmdtable = {
"convert":
(_convert,
@@ -439,4 +448,9 @@
('r', 'rev', '', 'import up to target revision REV'),
('', 'datesort', None, 'try to sort changesets by date')],
'hg convert [OPTION]... SOURCE [DEST [MAPFILE]]'),
+ "debug-svn-log":
+ (debugsvnlog,
+ [],
+ 'hg debug-svn-log'),
}
+
--- a/hgext/convert/common.py Thu Aug 02 23:38:32 2007 +0200
+++ b/hgext/convert/common.py Mon Aug 06 21:49:26 2007 +0200
@@ -1,4 +1,19 @@
# common code for the convert extension
+import base64
+import cPickle as pickle
+
+def encodeargs(args):
+ def encodearg(s):
+ lines = base64.encodestring(s)
+ lines = [l.splitlines()[0] for l in lines]
+ return ''.join(lines)
+
+ s = pickle.dumps(args)
+ return encodearg(s)
+
+def decodeargs(s):
+ s = base64.decodestring(s)
+ return pickle.loads(s)
class NoRepo(Exception): pass
@@ -118,3 +133,4 @@
"""Put tags into sink.
tags: {tagname: sink_rev_id, ...}"""
raise NotImplementedError()
+
--- a/hgext/convert/subversion.py Thu Aug 02 23:38:32 2007 +0200
+++ b/hgext/convert/subversion.py Mon Aug 06 21:49:26 2007 +0200
@@ -24,7 +24,7 @@
from cStringIO import StringIO
-from common import NoRepo, commit, converter_source
+from common import NoRepo, commit, converter_source, encodeargs
try:
from svn.core import SubversionException, Pool
@@ -58,6 +58,30 @@
self.copyfrom_rev = p.copyfrom_rev
self.action = p.action
+def get_log_child(fp, url, paths, start, end, limit=0, discover_changed_paths=True,
+ strict_node_history=False):
+ protocol = -1
+ def receiver(orig_paths, revnum, author, date, message, pool):
+ if orig_paths is not None:
+ for k, v in orig_paths.iteritems():
+ orig_paths[k] = changedpath(v)
+ pickle.dump((orig_paths, revnum, author, date, message),
+ fp, protocol)
+
+ try:
+ # Use an ra of our own so that our parent can consume
+ # our results without confusing the server.
+ t = transport.SvnRaTransport(url=url)
+ svn.ra.get_log(t.ra, paths, start, end, limit,
+ discover_changed_paths,
+ strict_node_history,
+ receiver)
+ except SubversionException, (_, num):
+ pickle.dump(num, fp, protocol)
+ else:
+ pickle.dump(None, fp, protocol)
+ fp.close()
+
# SVN conversion code stolen from bzr-svn and tailor
class convert_svn(converter_source):
def __init__(self, ui, url, rev=None):
@@ -196,34 +220,6 @@
def get_log(self, paths, start, end, limit=0, discover_changed_paths=True,
strict_node_history=False):
- '''wrapper for svn.ra.get_log.
- on a large repository, svn.ra.get_log pins huge amounts of
- memory that cannot be recovered. work around it by forking
- and writing results over a pipe.'''
-
- def child(fp):
- protocol = -1
- def receiver(orig_paths, revnum, author, date, message, pool):
- if orig_paths is not None:
- for k, v in orig_paths.iteritems():
- orig_paths[k] = changedpath(v)
- pickle.dump((orig_paths, revnum, author, date, message),
- fp, protocol)
-
- try:
- # Use an ra of our own so that our parent can consume
- # our results without confusing the server.
- t = transport.SvnRaTransport(url=self.url)
- svn.ra.get_log(t.ra, paths, start, end, limit,
- discover_changed_paths,
- strict_node_history,
- receiver)
- except SubversionException, (_, num):
- self.ui.print_exc()
- pickle.dump(num, fp, protocol)
- else:
- pickle.dump(None, fp, protocol)
- fp.close()
def parent(fp):
while True:
@@ -235,20 +231,19 @@
break
raise SubversionException("child raised exception", entry)
yield entry
+
+ args = [self.url, paths, start, end, limit, discover_changed_paths,
+ strict_node_history]
+ arg = encodeargs(args)
+ hgexe = util.hgexecutable()
+ cmd = '"%s "debug-svn-log""' % util.shellquote(hgexe)
+ stdin, stdout = os.popen2(cmd, 'b')
+
+ stdin.write(arg)
+ stdin.close()
- rfd, wfd = os.pipe()
- pid = os.fork()
- if pid:
- os.close(wfd)
- for p in parent(os.fdopen(rfd, 'rb')):
- yield p
- ret = os.waitpid(pid, 0)[1]
- if ret:
- raise util.Abort(_('get_log %s') % util.explain_exit(ret))
- else:
- os.close(rfd)
- child(os.fdopen(wfd, 'wb'))
- os._exit(0)
+ for p in parent(stdout):
+ yield p
def gettags(self):
tags = {}