comparison mercurial/changegroup.py @ 38908:1469584ad5fe

changegroup: specify ellipses mode explicitly

Currently, code throughout changegroup relies on the presence
of self._full_nodes to enable ellipses mode. This is a very tenuous
check. And the check may be wrong once we move _full_nodes into
cgpacker.

Let's capture the enabling of ellipses mode explicitly as a constructor
argument and as an instance variable.

We could probably derive ellipses mode by presence of other
variables. But for now, this explicit approach seems simplest
since it is most compatible with existing code.

Differential Revision: https://phab.mercurial-scm.org/D4090

38907:ad4c4cc9a5ac

    deltachunks = attr.ib()

class cgpacker(object):
    def __init__(self, repo, filematcher, version, allowreorder,
                 useprevdelta, builddeltaheader, manifestsend,
                 sendtreemanifests, bundlecaps=None, shallow=False,
                 ellipsisroots=None):
        """Given a source repo, construct a bundler.

        filematcher is a matcher that matches on files to include in the
        changegroup. Used to facilitate sparse changegroups.

        delta.

        manifestsend is a chunk to send after manifests have been fully emitted.

        sendtreemanifests indicates whether tree manifests should be emitted.

        bundlecaps is optional and can be used to specify the set of
        capabilities which can be used to build the bundle. While bundlecaps is
        unused in core Mercurial, extensions rely on this feature to communicate
        capabilities to customize the changegroup packer.

        self.version = version
        self._useprevdelta = useprevdelta
        self._builddeltaheader = builddeltaheader
        self._manifestsend = manifestsend
        self._sendtreemanifests = sendtreemanifests

        # Set of capabilities we can use to build the bundle.
        if bundlecaps is None:
            bundlecaps = set()
        self._bundlecaps = bundlecaps

        # The one invariant we *know* holds is that the new (potentially
        # bogus) DAG shape will be valid if we order the nodes in the
        # order that they're introduced in dramatis personae by the
        # changelog, so what we do is we sort the non-changelog histories
        # by the order in which they are used by the changelog.
        if util.safehasattr(self, '_full_nodes') and self._clnodetorev:
            key = lambda n: self._clnodetorev[lookup(n)]
            return [store.rev(n) for n in sorted(nodelist, key=key)]

        # for generaldelta revlogs, we linearize the revs; this will both be
        # much quicker and generate a much smaller bundle

        mfl = repo.manifestlog
        # TODO violates storage abstraction.
        mfrevlog = mfl._revlog
        changedfiles = set()

        ellipsesmode = util.safehasattr(self, '_full_nodes')

        # Callback for the changelog, used to collect changed files and
        # manifest nodes.
        # Returns the linkrev node (identity in the changelog case).
        def lookupcl(x):
            c = cl.read(x)
            clrevorder[x] = len(clrevorder)

            if ellipsesmode:
                # Only update mfs if x is going to be sent. Otherwise we
                # end up with bogus linkrevs specified for manifests and
                # we skip some manifest nodes that we should otherwise
                # have sent.
                if (x in self._full_nodes

        for chunk in self.generatemanifests(commonrevs, clrevorder,
                fastpathlinkrev, mfs, fnodes, source):
            yield chunk

        if ellipsesmode:
            mfdicts = None
            if self._isshallow:
                mfdicts = [(self._repo.manifestlog[n].read(), lr)
                           for (n, lr) in mfs.iteritems()]

                llr = filerevlog.linkrev
                fln = filerevlog.node
                revs = ((r, llr(r)) for r in filerevlog)
                return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)

        if ellipsesmode:
            # We need to pass the mfdicts variable down into
            # generatefiles(), but more than one command might have
            # wrapped generatefiles so we can't modify the function
            # signature. Instead, we pass the data to ourselves using an
            # instance attribute. I'm sorry.

                raise error.ProgrammingError(
                    'cg1 should not be used in this case')
            return prev

        # Narrow ellipses mode.
        if util.safehasattr(self, '_full_nodes'):
            # TODO: send better deltas when in narrow mode.
            #
            # changegroup.group() loops over revisions to send,
            # including revisions we'll skip. What this means is that
            # `prev` will be a potentially useless delta base for all

            base = nullrev

        return base

    def _revchunk(self, store, rev, prev, linknode):
        if util.safehasattr(self, '_full_nodes'):
            fn = self._revisiondeltanarrow
        else:
            fn = self._revisiondeltanormal

        delta = fn(store, rev, prev, linknode)

            linknode=linknode,
            flags=flags,
            deltachunks=(diffheader, data),
        )

def _makecg1packer(repo, filematcher, bundlecaps, shallow=False,
                   ellipsisroots=None):
    builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.linknode)

    return cgpacker(repo, filematcher, b'01',
                    useprevdelta=True,
                    allowreorder=None,
                    builddeltaheader=builddeltaheader,
                    manifestsend=b'',
                    sendtreemanifests=False,
                    bundlecaps=bundlecaps,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots)

def _makecg2packer(repo, filematcher, bundlecaps, shallow=False,
                   ellipsisroots=None):
    builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.basenode, d.linknode)

    # Since generaldelta is directly supported by cg2, reordering
    # generally doesn't help, so we disable it by default (treating

                    allowreorder=False,
                    builddeltaheader=builddeltaheader,
                    manifestsend=b'',
                    sendtreemanifests=False,
                    bundlecaps=bundlecaps,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots)

def _makecg3packer(repo, filematcher, bundlecaps, shallow=False,
                   ellipsisroots=None):
    builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)

    return cgpacker(repo, filematcher, b'03',
                    useprevdelta=False,
                    allowreorder=False,
                    builddeltaheader=builddeltaheader,
                    manifestsend=closechunk(),
                    sendtreemanifests=True,
                    bundlecaps=bundlecaps,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots)

_packermap = {'01': (_makecg1packer, cg1unpacker),
             # cg2 adds support for exchanging generaldelta

        versions.discard('01')
    assert versions
    return min(versions)

def getbundler(version, repo, bundlecaps=None, filematcher=None,
               shallow=False, ellipsisroots=None):
    assert version in supportedoutgoingversions(repo)

    if filematcher is None:
        filematcher = matchmod.alwaysmatcher(repo.root, '')

    if version == '01' and not filematcher.always():
        raise error.ProgrammingError('version 01 changegroups do not support '
                                     'sparse file matchers')

    # Requested files could include files not in the local store. So
    # filter those out.
    filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
                                             filematcher)

    fn = _packermap[version][0]
    return fn(repo, filematcher, bundlecaps, shallow=shallow,
              ellipsisroots=ellipsisroots)

def getunbundler(version, fh, alg, extras=None):
    return _packermap[version][1](fh, alg, extras=extras)

def _changegroupinfo(repo, nodes, source):

    return revisions, files

def _packellipsischangegroup(repo, common, match, relevant_nodes,
                             ellipsisroots, visitnodes, depth, source, version):
    if version in ('01', '02'):
        raise error.Abort(
            'ellipsis nodes require at least cg3 on client and server, '
            'but negotiated version %s' % version)
    # We wrap cg1packer.revchunk, using a side channel to pass
    # relevant_nodes into that area. Then if linknode isn't in the
    # set, we know we have an ellipsis node and we should defer
    # sending that node's data. We override close() to detect
    # pending ellipsis nodes and flush them.
    packer = getbundler(version, repo, filematcher=match,
                        shallow=depth is not None,
                        ellipsisroots=ellipsisroots)
    # Give the packer the list of nodes which should not be
    # ellipsis nodes. We store this rather than the set of nodes
    # that should be an ellipsis because for very large histories

38908:1469584ad5fe

    deltachunks = attr.ib()

class cgpacker(object):
    def __init__(self, repo, filematcher, version, allowreorder,
                 useprevdelta, builddeltaheader, manifestsend,
                 sendtreemanifests, bundlecaps=None, ellipses=False,
                 shallow=False, ellipsisroots=None):
        """Given a source repo, construct a bundler.

        filematcher is a matcher that matches on files to include in the
        changegroup. Used to facilitate sparse changegroups.

        delta.

        manifestsend is a chunk to send after manifests have been fully emitted.

        sendtreemanifests indicates whether tree manifests should be emitted.

        ellipses indicates whether ellipsis serving mode is enabled.

        bundlecaps is optional and can be used to specify the set of
        capabilities which can be used to build the bundle. While bundlecaps is
        unused in core Mercurial, extensions rely on this feature to communicate
        capabilities to customize the changegroup packer.

        self.version = version
        self._useprevdelta = useprevdelta
        self._builddeltaheader = builddeltaheader
        self._manifestsend = manifestsend
        self._sendtreemanifests = sendtreemanifests
        self._ellipses = ellipses

        # Set of capabilities we can use to build the bundle.
        if bundlecaps is None:
            bundlecaps = set()
        self._bundlecaps = bundlecaps

        # The one invariant we *know* holds is that the new (potentially
        # bogus) DAG shape will be valid if we order the nodes in the
        # order that they're introduced in dramatis personae by the
        # changelog, so what we do is we sort the non-changelog histories
        # by the order in which they are used by the changelog.
        if self._ellipses and self._clnodetorev:
            key = lambda n: self._clnodetorev[lookup(n)]
            return [store.rev(n) for n in sorted(nodelist, key=key)]

        # for generaldelta revlogs, we linearize the revs; this will both be
        # much quicker and generate a much smaller bundle

        mfl = repo.manifestlog
        # TODO violates storage abstraction.
        mfrevlog = mfl._revlog
        changedfiles = set()

        # Callback for the changelog, used to collect changed files and
        # manifest nodes.
        # Returns the linkrev node (identity in the changelog case).
        def lookupcl(x):
            c = cl.read(x)
            clrevorder[x] = len(clrevorder)

            if self._ellipses:
                # Only update mfs if x is going to be sent. Otherwise we
                # end up with bogus linkrevs specified for manifests and
                # we skip some manifest nodes that we should otherwise
                # have sent.
                if (x in self._full_nodes

        for chunk in self.generatemanifests(commonrevs, clrevorder,
                fastpathlinkrev, mfs, fnodes, source):
            yield chunk

        if self._ellipses:
            mfdicts = None
            if self._isshallow:
                mfdicts = [(self._repo.manifestlog[n].read(), lr)
                           for (n, lr) in mfs.iteritems()]

                llr = filerevlog.linkrev
                fln = filerevlog.node
                revs = ((r, llr(r)) for r in filerevlog)
                return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)

        if self._ellipses:
            # We need to pass the mfdicts variable down into
            # generatefiles(), but more than one command might have
            # wrapped generatefiles so we can't modify the function
            # signature. Instead, we pass the data to ourselves using an
            # instance attribute. I'm sorry.

                raise error.ProgrammingError(
                    'cg1 should not be used in this case')
            return prev

        # Narrow ellipses mode.
        if self._ellipses:
            # TODO: send better deltas when in narrow mode.
            #
            # changegroup.group() loops over revisions to send,
            # including revisions we'll skip. What this means is that
            # `prev` will be a potentially useless delta base for all

            base = nullrev

        return base

    def _revchunk(self, store, rev, prev, linknode):
        if self._ellipses:
            fn = self._revisiondeltanarrow
        else:
            fn = self._revisiondeltanormal

        delta = fn(store, rev, prev, linknode)

            linknode=linknode,
            flags=flags,
            deltachunks=(diffheader, data),
        )

def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
                   shallow=False, ellipsisroots=None):
    builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.linknode)

    return cgpacker(repo, filematcher, b'01',
                    useprevdelta=True,
                    allowreorder=None,
                    builddeltaheader=builddeltaheader,
                    manifestsend=b'',
                    sendtreemanifests=False,
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots)

def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
                   shallow=False, ellipsisroots=None):
    builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.basenode, d.linknode)

    # Since generaldelta is directly supported by cg2, reordering
    # generally doesn't help, so we disable it by default (treating

                    allowreorder=False,
                    builddeltaheader=builddeltaheader,
                    manifestsend=b'',
                    sendtreemanifests=False,
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots)

def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
                   shallow=False, ellipsisroots=None):
    builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
        d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)

    return cgpacker(repo, filematcher, b'03',
                    useprevdelta=False,
                    allowreorder=False,
                    builddeltaheader=builddeltaheader,
                    manifestsend=closechunk(),
                    sendtreemanifests=True,
                    bundlecaps=bundlecaps,
                    ellipses=ellipses,
                    shallow=shallow,
                    ellipsisroots=ellipsisroots)

_packermap = {'01': (_makecg1packer, cg1unpacker),
             # cg2 adds support for exchanging generaldelta

        versions.discard('01')
    assert versions
    return min(versions)

def getbundler(version, repo, bundlecaps=None, filematcher=None,
               ellipses=False, shallow=False, ellipsisroots=None):
    assert version in supportedoutgoingversions(repo)

    if filematcher is None:
        filematcher = matchmod.alwaysmatcher(repo.root, '')

    if version == '01' and not filematcher.always():
        raise error.ProgrammingError('version 01 changegroups do not support '
                                     'sparse file matchers')

    if ellipses and version in (b'01', b'02'):
        raise error.Abort(
            _('ellipsis nodes require at least cg3 on client and server, '
              'but negotiated version %s') % version)

    # Requested files could include files not in the local store. So
    # filter those out.
    filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
                                             filematcher)

    fn = _packermap[version][0]
    return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
              shallow=shallow, ellipsisroots=ellipsisroots)

def getunbundler(version, fh, alg, extras=None):
    return _packermap[version][1](fh, alg, extras=extras)

def _changegroupinfo(repo, nodes, source):

    return revisions, files

def _packellipsischangegroup(repo, common, match, relevant_nodes,
                             ellipsisroots, visitnodes, depth, source, version):
    # We wrap cg1packer.revchunk, using a side channel to pass
    # relevant_nodes into that area. Then if linknode isn't in the
    # set, we know we have an ellipsis node and we should defer
    # sending that node's data. We override close() to detect
    # pending ellipsis nodes and flush them.
    packer = getbundler(version, repo, filematcher=match,
                        ellipses=True,
                        shallow=depth is not None,
                        ellipsisroots=ellipsisroots)
    # Give the packer the list of nodes which should not be
    # ellipsis nodes. We store this rather than the set of nodes
    # that should be an ellipsis because for very large histories