annotate hgext/narrow/narrowchangegroup.py @ 36467:2d82a24d0c78

narrow: consider both local and remote matchers in narrowchangegroup

The existing code that picked one or the other seemed very
suspicious. This patch makes us intersect the local matcher with the
matcher from the remote, which seems better. It fixes one test case
and makes another one that used to crash no longer crash, but instead
silently succeed with a push that's lossy, so that remains to be
fixed.

The real reason for doing this now is that I'm going to move
narrowrepo.narrowmatch() onto localrepo and then it will always be
defined, which would otherwise break this code.

Differential Revision: https://phab.mercurial-scm.org/D2490

# narrowchangegroup.py - narrow clone changegroup creation and consumption
#
# Copyright 2017 Google, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

from mercurial.i18n import _
from mercurial import (
    changegroup,
    error,
    extensions,
    manifest,
    match as matchmod,
    mdiff,
    node,
    revlog,
    util,
)

def setup():

    def _cgmatcher(cgpacker):
        localmatcher = getattr(cgpacker._repo, 'narrowmatch', lambda: None)()
        remotematcher = getattr(cgpacker, '_narrow_matcher', lambda: None)()
        if localmatcher and remotematcher:
            return matchmod.intersectmatchers(localmatcher, remotematcher)
        else:
            return localmatcher or remotematcher

    def prune(orig, self, revlog, missing, commonrevs):
        if isinstance(revlog, manifest.manifestrevlog):
            matcher = _cgmatcher(self)
            if (matcher and
                not matcher.visitdir(revlog._dir[:-1] or '.')):
                return []
        return orig(self, revlog, missing, commonrevs)

    extensions.wrapfunction(changegroup.cg1packer, 'prune', prune)

    def generatefiles(orig, self, changedfiles, linknodes, commonrevs,
                      source):
        matcher = _cgmatcher(self)
        if matcher:
            changedfiles = filter(matcher, changedfiles)
        if getattr(self, 'is_shallow', False):
            # See comment in generate() for why this sadness is a thing.
            mfdicts = self._mfdicts
            del self._mfdicts
            # In a shallow clone, the linknodes callback needs to also include
            # those file nodes that are in the manifests we sent but weren't
            # introduced by those manifests.
            commonctxs = [self._repo[c] for c in commonrevs]
            oldlinknodes = linknodes
            clrev = self._repo.changelog.rev
            def linknodes(flog, fname):
                for c in commonctxs:
                    try:
                        fnode = c.filenode(fname)
                        self.clrev_to_localrev[c.rev()] = flog.rev(fnode)
                    except error.ManifestLookupError:
                        pass
                links = oldlinknodes(flog, fname)
                if len(links) != len(mfdicts):
                    for mf, lr in mfdicts:
                        fnode = mf.get(fname, None)
                        if fnode in links:
                            links[fnode] = min(links[fnode], lr, key=clrev)
                        elif fnode:
                            links[fnode] = lr
                return links
        return orig(self, changedfiles, linknodes, commonrevs, source)
    extensions.wrapfunction(
        changegroup.cg1packer, 'generatefiles', generatefiles)

    def ellipsisdata(packer, rev, revlog_, p1, p2, data, linknode):
        n = revlog_.node(rev)
        p1n, p2n = revlog_.node(p1), revlog_.node(p2)
        flags = revlog_.flags(rev)
        flags |= revlog.REVIDX_ELLIPSIS
        meta = packer.builddeltaheader(
            n, p1n, p2n, node.nullid, linknode, flags)
        # TODO: try and actually send deltas for ellipsis data blocks
        diffheader = mdiff.trivialdiffheader(len(data))
        l = len(meta) + len(diffheader) + len(data)
        return ''.join((changegroup.chunkheader(l),
                        meta,
                        diffheader,
                        data))

    def close(orig, self):
        getattr(self, 'clrev_to_localrev', {}).clear()
        if getattr(self, 'next_clrev_to_localrev', {}):
            self.clrev_to_localrev = self.next_clrev_to_localrev
            del self.next_clrev_to_localrev
        self.changelog_done = True
        return orig(self)
    extensions.wrapfunction(changegroup.cg1packer, 'close', close)

    # In a perfect world, we'd generate better ellipsis-ified graphs
    # for non-changelog revlogs. In practice, we haven't started doing
    # that yet, so the resulting DAGs for the manifestlog and filelogs
    # are actually full of bogus parentage on all the ellipsis
    # nodes. This has the side effect that, while the contents are
    # correct, the individual DAGs might be completely out of whack in
    # a case like 882681bc3166 and its ancestors (back about 10
    # revisions or so) in the main hg repo.
    #
    # The one invariant we *know* holds is that the new (potentially
    # bogus) DAG shape will be valid if we order the nodes in the
    # order that they're introduced in dramatis personae by the
    # changelog, so what we do is we sort the non-changelog histories
    # by the order in which they are used by the changelog.
    def _sortgroup(orig, self, revlog, nodelist, lookup):
        if not util.safehasattr(self, 'full_nodes') or not self.clnode_to_rev:
            return orig(self, revlog, nodelist, lookup)
        key = lambda n: self.clnode_to_rev[lookup(n)]
        return [revlog.rev(n) for n in sorted(nodelist, key=key)]

    extensions.wrapfunction(changegroup.cg1packer, '_sortgroup', _sortgroup)

    def generate(orig, self, commonrevs, clnodes, fastpathlinkrev, source):
        '''yield a sequence of changegroup chunks (strings)'''
        # Note: other than delegating to orig, the only deviation in
        # logic from normal hg's generate is marked with BEGIN/END
        # NARROW HACK.
        if not util.safehasattr(self, 'full_nodes'):
            # not sending a narrow bundle
            for x in orig(self, commonrevs, clnodes, fastpathlinkrev, source):
                yield x
            return

        repo = self._repo
        cl = repo.changelog
        mfl = repo.manifestlog
        mfrevlog = mfl._revlog

        clrevorder = {}
        mfs = {} # needed manifests
        fnodes = {} # needed file nodes
        changedfiles = set()

        # Callback for the changelog, used to collect changed files and manifest
        # nodes.
        # Returns the linkrev node (identity in the changelog case).
        def lookupcl(x):
            c = cl.read(x)
            clrevorder[x] = len(clrevorder)
            # BEGIN NARROW HACK
            #
            # Only update mfs if x is going to be sent. Otherwise we
            # end up with bogus linkrevs specified for manifests and
            # we skip some manifest nodes that we should otherwise
            # have sent.
            if x in self.full_nodes or cl.rev(x) in self.precomputed_ellipsis:
                n = c[0]
                # record the first changeset introducing this manifest version
                mfs.setdefault(n, x)
                # Set this narrow-specific dict so we have the lowest manifest
                # revnum to look up for this cl revnum. (Part of mapping
                # changelog ellipsis parents to manifest ellipsis parents)
                self.next_clrev_to_localrev.setdefault(cl.rev(x),
                                                       mfrevlog.rev(n))
            # We can't trust the changed files list in the changeset if the
            # client requested a shallow clone.
            if self.is_shallow:
                changedfiles.update(mfl[c[0]].read().keys())
            else:
                changedfiles.update(c[3])
            # END NARROW HACK
            # Record a complete list of potentially-changed files in
            # this manifest.
            return x

        self._verbosenote(_('uncompressed size of bundle content:\n'))
        size = 0
        for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
            size += len(chunk)
            yield chunk
        self._verbosenote(_('%8.i (changelog)\n') % size)

        # We need to make sure that the linkrev in the changegroup refers to
        # the first changeset that introduced the manifest or file revision.
        # The fastpath is usually safer than the slowpath, because the filelogs
        # are walked in revlog order.
        #
        # When taking the slowpath with reorder=None and the manifest revlog
        # uses generaldelta, the manifest may be walked in the "wrong" order.
        # Without 'clrevorder', we would get an incorrect linkrev (see fix in
        # cc0ff93d0c0c).
        #
        # When taking the fastpath, we are only vulnerable to reordering
        # of the changelog itself. The changelog never uses generaldelta, so
        # it is only reordered when reorder=True. To handle this case, we
        # simply take the slowpath, which already has the 'clrevorder' logic.
        # This was also fixed in cc0ff93d0c0c.
        fastpathlinkrev = fastpathlinkrev and not self._reorder
        # Treemanifests don't work correctly with fastpathlinkrev
        # either, because we don't discover which directory nodes to
        # send along with files. This could probably be fixed.
        fastpathlinkrev = fastpathlinkrev and (
            'treemanifest' not in repo.requirements)
        # Shallow clones also don't work correctly with fastpathlinkrev
        # because file nodes may need to be sent for a manifest even if they
        # weren't introduced by that manifest.
        fastpathlinkrev = fastpathlinkrev and not self.is_shallow

        for chunk in self.generatemanifests(commonrevs, clrevorder,
                fastpathlinkrev, mfs, fnodes, source):
            yield chunk
        # BEGIN NARROW HACK
        mfdicts = None
        if self.is_shallow:
            mfdicts = [(self._repo.manifestlog[n].read(), lr)
                       for (n, lr) in mfs.iteritems()]
        # END NARROW HACK
        mfs.clear()
        clrevs = set(cl.rev(x) for x in clnodes)

        if not fastpathlinkrev:
            def linknodes(unused, fname):
                return fnodes.get(fname, {})
        else:
            cln = cl.node
            def linknodes(filerevlog, fname):
                llr = filerevlog.linkrev
                fln = filerevlog.node
                revs = ((r, llr(r)) for r in filerevlog)
                return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)

        # BEGIN NARROW HACK
        #
        # We need to pass the mfdicts variable down into
        # generatefiles(), but more than one command might have
        # wrapped generatefiles so we can't modify the function
        # signature. Instead, we pass the data to ourselves using an
        # instance attribute. I'm sorry.
        self._mfdicts = mfdicts
        # END NARROW HACK
        for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
                                        source):
            yield chunk

        yield self.close()

        if clnodes:
            repo.hook('outgoing', node=node.hex(clnodes[0]), source=source)
    extensions.wrapfunction(changegroup.cg1packer, 'generate', generate)

    def revchunk(orig, self, revlog, rev, prev, linknode):
        if not util.safehasattr(self, 'full_nodes'):
            # not sending a narrow changegroup
            for x in orig(self, revlog, rev, prev, linknode):
                yield x
            return
        # build up some mapping information that's useful later. See
        # the local() nested function below.
        if not self.changelog_done:
            self.clnode_to_rev[linknode] = rev
            linkrev = rev
            self.clrev_to_localrev[linkrev] = rev
        else:
            linkrev = self.clnode_to_rev[linknode]
            self.clrev_to_localrev[linkrev] = rev
        # This is a node to send in full, because the changeset it
        # corresponds to was a full changeset.
        if linknode in self.full_nodes:
            for x in orig(self, revlog, rev, prev, linknode):
                yield x
            return
        # At this point, a node can either be one we should skip or an
        # ellipsis. If it's not an ellipsis, bail immediately.
        if linkrev not in self.precomputed_ellipsis:
            return
        linkparents = self.precomputed_ellipsis[linkrev]
        def local(clrev):
            """Turn a changelog revnum into a local revnum.

            The ellipsis dag is stored as revnums on the changelog,
            but when we're producing ellipsis entries for
            non-changelog revlogs, we need to turn those numbers into
            something local. This does that for us, and during the
            changelog sending phase will also expand the stored
            mappings as needed.
            """
            if clrev == node.nullrev:
                return node.nullrev
            if not self.changelog_done:
                # If we're doing the changelog, it's possible that we
                # have a parent that is already on the client, and we
                # need to store some extra mapping information so that
                # our contained ellipsis nodes will be able to resolve
                # their parents.
                if clrev not in self.clrev_to_localrev:
                    clnode = revlog.node(clrev)
                    self.clnode_to_rev[clnode] = clrev
                return clrev
            # Walk the ellipsis-ized changelog breadth-first looking for a
            # change that has been linked from the current revlog.
            #
            # For a flat manifest revlog only a single step should be necessary
            # as all relevant changelog entries are relevant to the flat
            # manifest.
            #
            # For a filelog or tree manifest dirlog however not every changelog
            # entry will have been relevant, so we need to skip some changelog
            # nodes even after ellipsis-izing.
            walk = [clrev]
            while walk:
                p = walk[0]
                walk = walk[1:]
                if p in self.clrev_to_localrev:
                    return self.clrev_to_localrev[p]
                elif p in self.full_nodes:
                    walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
                                    if pp != node.nullrev])
                elif p in self.precomputed_ellipsis:
                    walk.extend([pp for pp in self.precomputed_ellipsis[p]
                                    if pp != node.nullrev])
                else:
                    # In this case, we've got an ellipsis with parents
                    # outside the current bundle (likely an
                    # incremental pull). We "know" that we can use the
                    # value of this same revlog at whatever revision
                    # is pointed to by linknode. "Know" is in scare
                    # quotes because I haven't done enough examination
                    # of edge cases to convince myself this is really
                    # a fact - it works for all the (admittedly
                    # thorough) cases in our testsuite, but I would be
                    # somewhat unsurprised to find a case in the wild
                    # where this breaks down a bit. That said, I don't
                    # know if it would hurt anything.
                    for i in xrange(rev, 0, -1):
                        if revlog.linkrev(i) == clrev:
                            return i
                    # We failed to resolve a parent for this node, so
                    # we crash the changegroup construction.
                    raise error.Abort(
                        'unable to resolve parent while packing %r %r'
                        ' for changeset %r' % (revlog.indexfile, rev, clrev))
            return node.nullrev

        if not linkparents or (
            revlog.parentrevs(rev) == (node.nullrev, node.nullrev)):
            p1, p2 = node.nullrev, node.nullrev
        elif len(linkparents) == 1:
            p1, = sorted(local(p) for p in linkparents)
            p2 = node.nullrev
        else:
            p1, p2 = sorted(local(p) for p in linkparents)
        yield ellipsisdata(
            self, rev, revlog, p1, p2, revlog.revision(rev), linknode)
    extensions.wrapfunction(changegroup.cg1packer, 'revchunk', revchunk)

    def deltaparent(orig, self, revlog, rev, p1, p2, prev):
        if util.safehasattr(self, 'full_nodes'):
            # TODO: send better deltas when in narrow mode.
            #
            # changegroup.group() loops over revisions to send,
            # including revisions we'll skip. What this means is that
            # `prev` will be a potentially useless delta base for all
            # ellipsis nodes, as the client likely won't have it. In
            # the future we should do bookkeeping about which nodes
            # have been sent to the client, and try to be
            # significantly smarter about delta bases. This is
            # slightly tricky because this same code has to work for
            # all revlogs, and we don't have the linkrev/linknode here.
            return p1
        return orig(self, revlog, rev, p1, p2, prev)
    extensions.wrapfunction(changegroup.cg2packer, 'deltaparent', deltaparent)