comparison mercurial/changegroup.py @ 40344:2c5835b4246b

narrow: when widening, don't include manifests the client already has

When widening, we already don't include the changelog (since
f1844a10ee19) and files that the client already has (since
c73c7653dfb9). However, we still include all manifests needed for the
new narrowspec. When using flat manifests, that means we resend all
the manifests even though the client necessarily has all of them. For
tree manifests, we unnecessarily resend the root manifests and any
subdirectory manifests that the client already has.

This patch makes it so we no longer resend manifests that the client
already has. It does so by passing an extra matcher to the changegroup
packer and it uses that for filtering out directories matching the old
matcher's visitdir(). For consistency between directories and files,
it also makes the filtering of files look at both old and new matcher
rather than passing in a diff matcher as we did before.

Differential Revision: https://phab.mercurial-scm.org/D4895

40343:a69d5823af6d

    if progress:
        progress.complete()

class cgpacker(object):
    def __init__(self, repo, filematcher, version,
                 builddeltaheader, manifestsend,
                 forcedeltaparentprev=False,
                 bundlecaps=None, ellipses=False,
                 shallow=False, ellipsisroots=None, fullnodes=None):
        """Given a source repo, construct a bundler.

        filematcher is a matcher that matches on files to include in the
        changegroup. Used to facilitate sparse changegroups.

        forcedeltaparentprev indicates whether delta parents must be against
        the previous revision in a delta group. This should only be used for
        compatibility with changegroup version 1.

        fullnodes is the set of changelog nodes which should not be ellipsis
        nodes. We store this rather than the set of nodes that should be
        ellipsis because for very large histories we expect this to be
        significantly smaller.
        """
        assert filematcher
        self._filematcher = filematcher

        self.version = version
        self._forcedeltaparentprev = forcedeltaparentprev
        self._builddeltaheader = builddeltaheader
        self._manifestsend = manifestsend

        while tmfnodes:
            tree, nodes = tmfnodes.popitem()
            store = mfl.getstorage(tree)

            if not self._filematcher.visitdir(store.tree[:-1] or '.'):
                # No nodes to send because this directory is out of
                # the client's view of the repository (probably
                # because of narrow clones).
                prunednodes = []
            else:

                topic=_('manifests'),
                clrevtolocalrev=clrevtolocalrev,
                fullclnodes=self._fullclnodes,
                precomputedellipsis=self._precomputedellipsis)

            yield tree, deltas

    def _prunemanifests(self, store, nodes, commonrevs):
        # This is split out as a separate method to allow filtering
        # commonrevs in extension code.
        #

        return [n for n in nodes if flr(frev(n)) not in commonrevs]

    # The 'source' parameter is useful for extensions
    def generatefiles(self, changedfiles, commonrevs, source,
                      mfdicts, fastpathlinkrev, fnodes, clrevs):
        changedfiles = list(filter(self._filematcher, changedfiles))

        if not fastpathlinkrev:
            def normallinknodes(unused, fname):
                return fnodes.get(fname, {})
        else:

            yield fname, deltas

        progress.complete()

def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
                   shallow=False, ellipsisroots=None, fullnodes=None):
    builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.linknode)

    return cgpacker(repo, filematcher, b'01',
                    builddeltaheader=builddeltaheader,
                    manifestsend=b'',
                    forcedeltaparentprev=True,
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots,
                    fullnodes=fullnodes)

def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
                   shallow=False, ellipsisroots=None, fullnodes=None):
    builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.basenode, d.linknode)

    return cgpacker(repo, filematcher, b'02',
                    builddeltaheader=builddeltaheader,
                    manifestsend=b'',
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots,
                    fullnodes=fullnodes)

def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
                   shallow=False, ellipsisroots=None, fullnodes=None):
    builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)

    return cgpacker(repo, filematcher, b'03',
                    builddeltaheader=builddeltaheader,
                    manifestsend=closechunk(),
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,

    if 'generaldelta' in repo.requirements:
        versions.discard('01')
    assert versions
    return min(versions)

def getbundler(version, repo, bundlecaps=None, filematcher=None,
               ellipses=False, shallow=False, ellipsisroots=None,
               fullnodes=None):
    assert version in supportedoutgoingversions(repo)

    if filematcher is None:
        filematcher = matchmod.alwaysmatcher(repo.root, '')

    if version == '01' and not filematcher.always():
        raise error.ProgrammingError('version 01 changegroups do not support '
                                     'sparse file matchers')

    if ellipses and version in (b'01', b'02'):
        raise error.Abort(
            _('ellipsis nodes require at least cg3 on client and server, '
              'but negotiated version %s') % version)

    # Requested files could include files not in the local store. So
    # filter those out.
    filematcher = repo.narrowmatch(filematcher)

    fn = _packermap[version][0]
    return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
              shallow=shallow, ellipsisroots=ellipsisroots,
              fullnodes=fullnodes)

def getunbundler(version, fh, alg, extras=None):
    return _packermap[version][1](fh, alg, extras=extras)

                          fastpath=fastpath, bundlecaps=bundlecaps)
    return getunbundler(version, util.chunkbuffer(cgstream), None,
                        {'clcount': len(outgoing.missing) })

def makestream(repo, outgoing, version, source, fastpath=False,
               bundlecaps=None, filematcher=None):
    bundler = getbundler(version, repo, bundlecaps=bundlecaps,
                         filematcher=filematcher)

    repo = repo.unfiltered()
    commonrevs = outgoing.common
    csets = outgoing.missing
    heads = outgoing.missingheads

40344:2c5835b4246b

    if progress:
        progress.complete()

class cgpacker(object):
    def __init__(self, repo, oldmatcher, matcher, version,
                 builddeltaheader, manifestsend,
                 forcedeltaparentprev=False,
                 bundlecaps=None, ellipses=False,
                 shallow=False, ellipsisroots=None, fullnodes=None):
        """Given a source repo, construct a bundler.

        oldmatcher is a matcher that matches on files the client already has.
        These will not be included in the changegroup.

        matcher is a matcher that matches on files to include in the
        changegroup. Used to facilitate sparse changegroups.

        forcedeltaparentprev indicates whether delta parents must be against
        the previous revision in a delta group. This should only be used for
        compatibility with changegroup version 1.

        fullnodes is the set of changelog nodes which should not be ellipsis
        nodes. We store this rather than the set of nodes that should be
        ellipsis because for very large histories we expect this to be
        significantly smaller.
        """
        assert oldmatcher
        assert matcher
        self._oldmatcher = oldmatcher
        self._matcher = matcher

        self.version = version
        self._forcedeltaparentprev = forcedeltaparentprev
        self._builddeltaheader = builddeltaheader
        self._manifestsend = manifestsend

        while tmfnodes:
            tree, nodes = tmfnodes.popitem()
            store = mfl.getstorage(tree)

            if not self._matcher.visitdir(store.tree[:-1] or '.'):
                # No nodes to send because this directory is out of
                # the client's view of the repository (probably
                # because of narrow clones).
                prunednodes = []
            else:

                topic=_('manifests'),
                clrevtolocalrev=clrevtolocalrev,
                fullclnodes=self._fullclnodes,
                precomputedellipsis=self._precomputedellipsis)

            if not self._oldmatcher.visitdir(store.tree[:-1] or '.'):
                yield tree, deltas
            else:
                # 'deltas' is a generator and we need to consume it even if
                # we are not going to send it because a side-effect is that
                # it updates tmdnodes (via lookupfn)
                for d in deltas:
                    pass
                if not tree:
                    yield tree, []

    def _prunemanifests(self, store, nodes, commonrevs):
        # This is split out as a separate method to allow filtering
        # commonrevs in extension code.
        #

        return [n for n in nodes if flr(frev(n)) not in commonrevs]

    # The 'source' parameter is useful for extensions
    def generatefiles(self, changedfiles, commonrevs, source,
                      mfdicts, fastpathlinkrev, fnodes, clrevs):
        changedfiles = [f for f in changedfiles
                        if self._matcher(f) and not self._oldmatcher(f)]

        if not fastpathlinkrev:
            def normallinknodes(unused, fname):
                return fnodes.get(fname, {})
        else:

            yield fname, deltas

        progress.complete()

def _makecg1packer(repo, oldmatcher, matcher, bundlecaps,
                   ellipses=False, shallow=False, ellipsisroots=None,
                   fullnodes=None):
    builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.linknode)

    return cgpacker(repo, oldmatcher, matcher, b'01',
                    builddeltaheader=builddeltaheader,
                    manifestsend=b'',
                    forcedeltaparentprev=True,
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots,
                    fullnodes=fullnodes)

def _makecg2packer(repo, oldmatcher, matcher, bundlecaps,
                   ellipses=False, shallow=False, ellipsisroots=None,
                   fullnodes=None):
    builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.basenode, d.linknode)

    return cgpacker(repo, oldmatcher, matcher, b'02',
                    builddeltaheader=builddeltaheader,
                    manifestsend=b'',
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots,
                    fullnodes=fullnodes)

def _makecg3packer(repo, oldmatcher, matcher, bundlecaps,
                   ellipses=False, shallow=False, ellipsisroots=None,
                   fullnodes=None):
    builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)

    return cgpacker(repo, oldmatcher, matcher, b'03',
                    builddeltaheader=builddeltaheader,
                    manifestsend=closechunk(),
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,

    if 'generaldelta' in repo.requirements:
        versions.discard('01')
    assert versions
    return min(versions)

def getbundler(version, repo, bundlecaps=None, oldmatcher=None,
               matcher=None, ellipses=False, shallow=False,
               ellipsisroots=None, fullnodes=None):
    assert version in supportedoutgoingversions(repo)

    if matcher is None:
        matcher = matchmod.alwaysmatcher(repo.root, '')
    if oldmatcher is None:
        oldmatcher = matchmod.nevermatcher(repo.root, '')

    if version == '01' and not matcher.always():
        raise error.ProgrammingError('version 01 changegroups do not support '
                                     'sparse file matchers')

    if ellipses and version in (b'01', b'02'):
        raise error.Abort(
            _('ellipsis nodes require at least cg3 on client and server, '
              'but negotiated version %s') % version)

    # Requested files could include files not in the local store. So
    # filter those out.
    matcher = repo.narrowmatch(matcher)

    fn = _packermap[version][0]
    return fn(repo, oldmatcher, matcher, bundlecaps, ellipses=ellipses,
              shallow=shallow, ellipsisroots=ellipsisroots,
              fullnodes=fullnodes)

def getunbundler(version, fh, alg, extras=None):
    return _packermap[version][1](fh, alg, extras=extras)

                          fastpath=fastpath, bundlecaps=bundlecaps)
    return getunbundler(version, util.chunkbuffer(cgstream), None,
                        {'clcount': len(outgoing.missing) })

def makestream(repo, outgoing, version, source, fastpath=False,
               bundlecaps=None, matcher=None):
    bundler = getbundler(version, repo, bundlecaps=bundlecaps,
                         matcher=matcher)

    repo = repo.unfiltered()
    commonrevs = outgoing.common
    csets = outgoing.missing
    heads = outgoing.missingheads