annotate hgext/hgfastobs.py @ 824:fed090e07621

exchange: pull markers relevant to the pulled subset only

With the command recently introduced we can select to pull only markers relevant
to some nodes. We are now pull all markers for all node in the relevant subset.
We'll try to pull less (just markers for node where local and remote marker
diverge) later, but we need some marker discovery mechanism for that. which are
not easy.

"""Extension to try and speed up transfer of obsolete markers.

Mercurial 2.6 transfers obsolete markers in the dumbest way possible:
it simply transfers all of them to the server on every
operation. While this /works/, it's not ideal because it's a large
amount of extra data for users to pull down (1.9M for the 17k obsolete
markers in hg-crew as of this writing in late July 2013). It's also
frustrating because this transfer takes a nontrivial amount of time.

You can specify a strategy with the config knob
obsolete.syncstrategy. Current strategies are "stock" and
"boxfill". Default strategy is presently boxfill.

:stock: use the default strategy of mercurial explaned above

:boxfill: transmit obsolete markers which list any of transmitted changesets as
          a successor (transitively), as well as any kill markers for dead
          nodes descended from any of the precursors of outgoing.missing.

TODO(durin42): consider better names for sync strategies.
"""
import sys

from mercurial import base85
from mercurial import commands
from mercurial import extensions
from mercurial import node
from mercurial import obsolete
from mercurial import exchange
from mercurial import revset
from mercurial.i18n import _

_strategies = {
    'stock': exchange._pushobsolete,
    }

def _strategy(name, default=False):
    def inner(func):
        _strategies[name] = func
        if default:
            _strategies[None] = func
        return func
    return inner

def _pushobsoletewrapper(orig, pushop):
    stratfn = _strategies[pushop.repo.ui.config('obsolete', 'syncstrategy')]
    return stratfn(pushop)

extensions.wrapfunction(exchange, '_pushobsolete', _pushobsoletewrapper)

def _precursors(repo, s):
    """Precursor of a changeset"""
    cs = set()
    nm = repo.changelog.nodemap
    markerbysubj = repo.obsstore.precursors
    for r in s:
        for p in markerbysubj.get(repo[r].node(), ()):
            pr = nm.get(p[0])
            if pr is not None:
                cs.add(pr)
    return cs

def _revsetprecursors(repo, subset, x):
    s = revset.getset(repo, revset.baseset(range(len(repo))), x)
    cs = _precursors(repo, s)
    return revset.baseset([r for r in subset if r in cs])

revset.symbols['_fastobs_precursors'] = _revsetprecursors


@_strategy('boxfill', default=True)
def boxfill(pushop):
    """The "fill in the box" strategy from the 2.6 sprint.

    See the notes[0] from the 2.6 sprint for what "fill in the box"
    means here. It's a fairly subtle algorithm, which may have
    surprising behavior at times, but was the least-bad option
    proposed at the sprint.

    [0]: https://bitbucket.org/durin42/2.6sprint-notes/src/tip/mercurial26-obsstore-rev.1398.txt
    """
    repo = pushop.repo
    remote = pushop.remote
    outgoing = pushop.outgoing
    urepo = pushop.repo.unfiltered()
    # need to collect obsolete markers which list any of
    # outgoing.missing as a successor (transitively), as well as any
    # kill markers for dead nodes descended from any of the precursors
    # of outgoing.missing.
    boxedges = urepo.revs(
        '(descendants(_fastobs_precursors(%ln)) or '
        ' descendants(%ln)) and hidden()',
        outgoing.missing, outgoing.missing)
    transmit = []
    for node in outgoing.missing:
        transmit.extend(obsolete.precursormarkers(urepo[node]))
    for rev in boxedges:
        transmit.extend(obsolete.successormarkers(urepo[rev]))
    transmit = list(set(transmit))
    xmit, total = len(transmit), len(repo.obsstore._all)
    repo.ui.status(
        'boxpush: about to transmit %d obsolete markers (%d markers total)\n'
        % (xmit, total))
    parts, size, chunk = [], 0, 0
    def transmitmarks():
            repo.ui.note(
                'boxpush: sending a chunk of obsolete markers\n')
            data = ''.join([obsolete._pack('>B', obsolete._fmversion)] + parts)
            remote.pushkey('obsolete', 'dump-%d' % chunk, '',
                           base85.b85encode(data))

    for marker in transmit:
        enc = obsolete._encodeonemarker(_markertuple(marker))
        parts.append(enc)
        size += len(enc)
        if size > obsolete._maxpayload:
            transmitmarks()
            parts, size = [], 0
            chunk += 1
    if parts:
        transmitmarks()

def _markertuple(marker):
    return marker._data