convert/mtn: add support for using monotone's "automate stdio" when available
Currently the convert extension spawns a new mtn process for each
operation. For a large repository, this ends up being hundreds of
thousands of processes. The following enables usage of monotone's
"automate stdio" functionality - documented at:
http://www.monotone.ca/docs/Automation.html#index-mtn-automate-stdio-188
The effect is that (after determining that a new enough mtn executable
is available) a single long-running mtn process is used for all the
operations, using stdin/stdout to send commands and read output.
This has a pretty significant effect on the performance of some parts
of the conversion process.
--- a/hgext/convert/monotone.py Thu Mar 24 16:54:09 2011 -0400
+++ b/hgext/convert/monotone.py Wed Mar 23 14:26:56 2011 -0400
@@ -19,6 +19,7 @@
self.ui = ui
self.path = path
+ self.automatestdio = False
norepo = NoRepo(_("%s does not look like a monotone repository")
% path)
@@ -73,9 +74,102 @@
self.rev = rev
def mtnrun(self, *args, **kwargs):
+ if self.automatestdio:
+ return self.mtnrunstdio(*args, **kwargs)
+ else:
+ return self.mtnrunsingle(*args, **kwargs)
+
+ def mtnrunsingle(self, *args, **kwargs):
kwargs['d'] = self.path
return self.run0('automate', *args, **kwargs)
+ def mtnrunstdio(self, *args, **kwargs):
+ # Prepare the command in automate stdio format
+ command = []
+ for k, v in kwargs.iteritems():
+ command.append("%s:%s" % (len(k), k))
+ if v:
+ command.append("%s:%s" % (len(v), v))
+ if command:
+ command.insert(0, 'o')
+ command.append('e')
+
+ command.append('l')
+ for arg in args:
+ command += "%s:%s" % (len(arg), arg)
+ command.append('e')
+ command = ''.join(command)
+
+ self.ui.debug("mtn: sending '%s'\n" % command)
+ self.mtnwritefp.write(command)
+ self.mtnwritefp.flush()
+
+ return self.mtnstdioreadcommandoutput(command)
+
+ def mtnstdioreadpacket(self):
+ read = None
+ commandnbr = ''
+ while read != ':':
+ read = self.mtnreadfp.read(1)
+ if not read:
+ raise util.Abort(_('bad mtn packet - no end of commandnbr'))
+ commandnbr += read
+ commandnbr = commandnbr[:-1]
+
+ stream = self.mtnreadfp.read(1)
+ if stream not in 'mewptl':
+ raise util.Abort(_('bad mtn packet - bad stream type %s' % stream))
+
+ read = self.mtnreadfp.read(1)
+ if read != ':':
+ raise util.Abort(_('bad mtn packet - no divider before size'))
+
+ read = None
+ lengthstr = ''
+ while read != ':':
+ read = self.mtnreadfp.read(1)
+ if not read:
+ raise util.Abort(_('bad mtn packet - no end of packet size'))
+ lengthstr += read
+ try:
+ length = long(lengthstr[:-1])
+ except TypeError:
+ raise util.Abort(_('bad mtn packet - bad packet size %s')
+ % lengthstr)
+
+ read = self.mtnreadfp.read(length)
+ if len(read) != length:
+ raise util.Abort(_("bad mtn packet - unable to read full packet "
+ "read %s of %s") % (len(read), length))
+
+ return (commandnbr, stream, length, read)
+
+ def mtnstdioreadcommandoutput(self, command):
+ retval = ''
+ while True:
+ commandnbr, stream, length, output = self.mtnstdioreadpacket()
+ self.ui.debug('mtn: read packet %s:%s:%s\n' %
+ (commandnbr, stream, length))
+
+ if stream == 'l':
+ # End of command
+ if output != '0':
+ raise util.Abort(_("mtn command '%s' returned %s") %
+ (command, output))
+ break
+ elif stream in 'ew':
+ # Error, warning output
+ self.ui.warn(_('%s error:\n') % self.command)
+ self.ui.warn(output)
+ elif stream == 'p':
+ # Progress messages
+ self.ui.debug('mtn: ' + output)
+ elif stream == 'm':
+ # Main stream - command output
+ retval = output
+
+ return retval
+
def mtnloadmanifest(self, rev):
if self.manifest_rev == rev:
return
@@ -225,3 +319,43 @@
# This function is only needed to support --filemap
# ... and we don't support that
raise NotImplementedError()
+
+ def before(self):
+ # Check if we have a new enough version to use automate stdio
+ version = 0.0
+ try:
+ versionstr = self.mtnrunsingle("interface_version")
+ version = float(versionstr)
+ except Exception:
+ raise util.Abort(_("unable to determine mtn automate interface "
+ "version"))
+
+ if version >= 12.0:
+ self.automatestdio = True
+ self.ui.debug("mtn automate version %s - using automate stdio\n" %
+ version)
+
+ # launch the long-running automate stdio process
+ self.mtnwritefp, self.mtnreadfp = self._run2('automate', 'stdio',
+ '-d', self.path)
+ # read the headers
+ read = self.mtnreadfp.readline()
+ if read != 'format-version: 2\n':
+ raise util.Abort(_('mtn automate stdio header unexpected: %s')
+ % read)
+ while read != '\n':
+ read = self.mtnreadfp.readline()
+ if not read:
+ raise util.Abort(_("failed to reach end of mtn automate "
+ "stdio headers"))
+ else:
+ self.ui.debug("mtn automate version %s - not using automate stdio "
+ "(automate >= 12.0 - mtn >= 0.46 is needed)\n" % version)
+
+ def after(self):
+ if self.automatestdio:
+ self.mtnwritefp.close()
+ self.mtnwritefp = None
+ self.mtnreadfp.close()
+ self.mtnreadfp = None
+