changeset 791:5f3b53d74b7f

fastobs: first commit of an extension to test obsolete marker exchange methods

This currently implements the "fill in the box" approach discussed at
the 2.6 sprint, which seems to handle common cases correctly with
significantly less data transferred.

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hgfastobs.py	Fri Jul 26 16:04:40 2013 -0400
@@ -0,0 +1,93 @@
+"""Extension to try and speed up transfer of obsolete markers.
+
+Mercurial 2.6 transfers obsolete markers in the dumbest way possible:
+it simply transfers all of them to the server on every
+operation. While this /works/, it's not ideal because it's a large
+amount of extra data for users to pull down (1.9M for the 17k obsolete
+markers in hg-crew as of this writing in late July 2013). It's also
+frustrating because this transfer takes a nontrivial amount of time.
+
+You can specify a strategy with the config knob
+obsolete.syncstrategy. Current strategies are "stock" and
+"boxfill". Default strategy is presently boxfill.
+
+TODO(durin42): consider better names for sync strategies.
+"""
+import sys
+
+from mercurial import commands
+from mercurial import extensions
+from mercurial import obsolete
+from mercurial import node
+from mercurial.i18n import _
+
+_strategies = {
+    'stock': obsolete.syncpush,
+    }
+
+def _strategy(name, default=False):
+    def inner(func):
+        _strategies[name] = func
+        if default:
+            _strategies[None] = func
+        return func
+    return inner
+
+def syncpushwrapper(orig, repo, remote):
+    stratfn = _strategies[repo.ui.config('obsolete', 'syncstrategy')]
+    return stratfn(repo, remote)
+
+extensions.wrapfunction(obsolete, 'syncpush', syncpushwrapper)
+
+def pushmarkerwrapper(orig, repo, *args):
+    if repo.ui.config('obsolete', 'syncstrategy') == 'stock':
+        return orig(repo, *args)
+    # We shouldn't need to do this, since we transmit markers
+    # effectively during push in localrepo. Just return success.
+    return 1
+
+def _getoutgoing():
+    f = sys._getframe(4)
+    return f.f_locals['outgoing']
+
+@_strategy('boxfill', default=True)
+def boxfill(repo, remote):
+    """The "fill in the box" strategy from the 2.6 sprint.
+
+    See the notes[0] from the 2.6 sprint for what "fill in the box"
+    means here. It's a fairly subtle algorithm, which may have
+    surprising behavior at times, but was the least-bad option
+    proposed at the sprint.
+
+    [0]: https://bitbucket.org/durin42/2.6sprint-notes/src/tip/mercurial26-obsstore-rev.1398.txt
+    """
+    outgoing = _getoutgoing()
+    urepo = repo.unfiltered()
+    # need to collect obsolete markers which list any of
+    # outgoing.missing as a successor (transitively), as well as any
+    # kill markers for dead nodes descended from any of the precursors
+    # of outgoing.missing.
+    boxedges = urepo.revs(
+        '(descendants(precursors(%ln)) or descendants(%ln)) and hidden()',
+        outgoing.missing, outgoing.missing)
+    transmit = []
+    for node in boxedges:
+        transmit.extend(obsolete.successormarkers(urepo[node]))
+    xmit, total = len(transmit), len(repo.obsstore._all)
+    repo.ui.status('about to transmit %d obsolete markers (%d markers total)\n'
+                   % (xmit, total))
+    parts, size, chunk = [], 0, 0
+    for marker in transmit:
+        enc = obsolete._encodeonemarker(_markertuple(marker))
+        parts.append(enc)
+        size += len(enc)
+        if size > obsolete._maxpayload:
+            repo.ui.note(
+                'obsolete boxpush: sending a chunk of obsolete markers\n')
+            data = ''.join([obsolete._pack('>B', _fmversion)], parts)
+            remote.pushkey('obsolete', 'dump%d' % chunk, base85.b85encode(data))
+            parts, size = [], 0
+            chunk += 1
+
+def _markertuple(marker):
+    return marker._data