comparison mercurial/changegroup.py @ 39001:a6e1ff40e335

changegroup: pass clrevtolocalrev to each group

clrevtolocalrev is a per-changegroup group mapping revisions to
aid with shallow clone.

Back when this functionality was implemented in an extension, this
dict was added to the packer instance so monkeypatched functions
could reference it there. Now that this code is part of core, we
can pass the dict to each consumer properly so it doesn't have to
live on the cgpacker instance. This commit does that.

Differential Revision: https://phab.mercurial-scm.org/D4193

39000:4cff37564006

        if self._repo.ui.verbose and not self._repo.ui.debugflag:
            self._verbosenote = self._repo.ui.note
        else:
            self._verbosenote = lambda s: None

        # Maps CL revs to per-revlog revisions. Cleared in close() at
        # the end of each group.
        self._clrevtolocalrev = {}

    def _close(self):
        # Ellipses serving mode.
        self._clrevtolocalrev.clear()

        return closechunk()

    def group(self, revs, store, ischangelog, lookup, units=None):
        """Calculate a delta group, yielding a sequence of changegroup chunks
        (strings).

        Given a list of changeset revs, return a set of deltas and
        metadata corresponding to nodes. The first delta is

            prev, curr = revs[r], revs[r + 1]
            linknode = lookup(store.node(curr))

            if self._ellipses:
                linkrev = cl.rev(linknode)
                self._clrevtolocalrev[linkrev] = curr

                # This is a node to send in full, because the changeset it
                # corresponds to was a full changeset.
                if linknode in self._fullclnodes:
                    delta = _revisiondeltanormal(store, curr, prev, linknode,
                                                 self._deltaparentfn)
                elif linkrev not in self._precomputedellipsis:
                    delta = None
                else:
                    delta = self._revisiondeltanarrow(store, ischangelog,
                                                      curr, linkrev, linknode)
            else:
                delta = _revisiondeltanormal(store, curr, prev, linknode,
                                             self._deltaparentfn)

            if not delta:

                return []

        rr, rl = store.rev, store.linkrev
        return [n for n in missing if rl(rr(n)) not in commonrevs]

    def _packmanifests(self, dir, dirlog, revs, lookuplinknode):
        """Pack manifests into a changegroup stream.

        Encodes the directory name in the output so multiple manifests
        can be sent. Multiple manifests is not supported by cg1 and cg2.
        """
        if dir:
            assert self.version == b'03'
            yield _fileheader(dir)

        for chunk in self.group(revs, dirlog, False, lookuplinknode,
                                units=_('manifests')):
            yield chunk

    def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
        """Yield a sequence of changegroup byte chunks."""

        self._verbosenote(_('%8.i (changelog)\n') % size)

        clrevorder = clstate['clrevorder']
        mfs = clstate['mfs']
        changedfiles = clstate['changedfiles']

        if self._ellipses:
            self._clrevtolocalrev = clstate['clrevtomanifestrev']

        # We need to make sure that the linkrev in the changegroup refers to
        # the first changeset that introduced the manifest or file revision.
        # The fastpath is usually safer than the slowpath, because the filelogs
        # are walked in revlog order.

            'treemanifest' not in repo.requirements)

        fnodes = {}  # needed file nodes

        for chunk in self.generatemanifests(commonrevs, clrevorder,
                fastpathlinkrev, mfs, fnodes, source):
            yield chunk

        mfdicts = None
        if self._ellipses and self._isshallow:
            mfdicts = [(self._repo.manifestlog[n].read(), lr)

            'mfs': mfs,
            'changedfiles': changedfiles,
            'clrevtomanifestrev': clrevtomanifestrev,
        }

        gen = self.group(revs, cl, True, lookupcl, units=_('changesets'))

        return state, gen

    def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
                          fnodes, source):
        """Returns an iterator of changegroup chunks containing manifests.

        `source` is unused here, but is used by extensions like remotefilelog to
        change what is sent based in pulls vs pushes, etc.
        """

                                              lookupfn)
                else:
                    revs = _sortnodesnormal(store, prunednodes,
                                            self._reorder)

                for x in self._packmanifests(dir, store, revs, lookupfn):
                    size += len(x)
                    yield x
        self._verbosenote(_('%8.i (manifests)\n') % size)
        yield self._manifestsend

                flinkrev = store.linkrev
                fnode = store.node
                revs = ((r, flinkrev(r)) for r in store)
                return dict((fnode(r), cln(lr))
                            for r, lr in revs if lr in clrevs)

        if self._isshallow:
            # In a shallow clone, the linknodes callback needs to also include
            # those file nodes that are in the manifests we sent but weren't
            # introduced by those manifests.

            # TODO have caller pass in appropriate function.
            def linknodes(flog, fname):
                for c in commonctxs:
                    try:
                        fnode = c.filenode(fname)
                        self._clrevtolocalrev[c.rev()] = flog.rev(fnode)
                    except error.ManifestLookupError:
                        pass
                links = normallinknodes(flog, fname)
                if len(links) != len(mfdicts):
                    for mf, lr in mfdicts:

            filerevlog = repo.file(fname)
            if not filerevlog:
                raise error.Abort(_("empty or missing file data for %s") %
                                  fname)

            linkrevnodes = linknodes(filerevlog, fname)
            # Lookup for filenodes, we collected the linkrev nodes above in the
            # fastpath case and with lookupmf in the slowpath case.
            def lookupfilelog(x):
                return linkrevnodes[x]

                progress.update(i + 1, item=fname)
                h = _fileheader(fname)
                size = len(h)
                yield h
                for chunk in self.group(revs, filerevlog, False, lookupfilelog):
                    size += len(chunk)
                    yield chunk
                self._verbosenote(_('%8.i  %s\n') % (size, fname))
        progress.complete()

    def _revisiondeltanarrow(self, store, ischangelog, rev, linkrev, linknode):
        linkparents = self._precomputedellipsis[linkrev]
        def local(clrev):
            """Turn a changelog revnum into a local revnum.

            The ellipsis dag is stored as revnums on the changelog,

            # nodes even after ellipsis-izing.
            walk = [clrev]
            while walk:
                p = walk[0]
                walk = walk[1:]
                if p in self._clrevtolocalrev:
                    return self._clrevtolocalrev[p]
                elif p in self._fullclnodes:
                    walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
                                    if pp != nullrev])
                elif p in self._precomputedellipsis:
                    walk.extend([pp for pp in self._precomputedellipsis[p]

39001:a6e1ff40e335

        if self._repo.ui.verbose and not self._repo.ui.debugflag:
            self._verbosenote = self._repo.ui.note
        else:
            self._verbosenote = lambda s: None

    def _close(self):
        return closechunk()

    def group(self, revs, store, ischangelog, lookup, units=None,
              clrevtolocalrev=None):
        """Calculate a delta group, yielding a sequence of changegroup chunks
        (strings).

        Given a list of changeset revs, return a set of deltas and
        metadata corresponding to nodes. The first delta is

            prev, curr = revs[r], revs[r + 1]
            linknode = lookup(store.node(curr))

            if self._ellipses:
                linkrev = cl.rev(linknode)
                clrevtolocalrev[linkrev] = curr

                # This is a node to send in full, because the changeset it
                # corresponds to was a full changeset.
                if linknode in self._fullclnodes:
                    delta = _revisiondeltanormal(store, curr, prev, linknode,
                                                 self._deltaparentfn)
                elif linkrev not in self._precomputedellipsis:
                    delta = None
                else:
                    delta = self._revisiondeltanarrow(store, ischangelog,
                                                      curr, linkrev, linknode,
                                                      clrevtolocalrev)
            else:
                delta = _revisiondeltanormal(store, curr, prev, linknode,
                                             self._deltaparentfn)

            if not delta:

                return []

        rr, rl = store.rev, store.linkrev
        return [n for n in missing if rl(rr(n)) not in commonrevs]

    def _packmanifests(self, dir, dirlog, revs, lookuplinknode,
                       clrevtolocalrev):
        """Pack manifests into a changegroup stream.

        Encodes the directory name in the output so multiple manifests
        can be sent. Multiple manifests is not supported by cg1 and cg2.
        """
        if dir:
            assert self.version == b'03'
            yield _fileheader(dir)

        for chunk in self.group(revs, dirlog, False, lookuplinknode,
                                units=_('manifests'),
                                clrevtolocalrev=clrevtolocalrev):
            yield chunk

    def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
        """Yield a sequence of changegroup byte chunks."""

        self._verbosenote(_('%8.i (changelog)\n') % size)

        clrevorder = clstate['clrevorder']
        mfs = clstate['mfs']
        changedfiles = clstate['changedfiles']

        # We need to make sure that the linkrev in the changegroup refers to
        # the first changeset that introduced the manifest or file revision.
        # The fastpath is usually safer than the slowpath, because the filelogs
        # are walked in revlog order.

            'treemanifest' not in repo.requirements)

        fnodes = {}  # needed file nodes

        for chunk in self.generatemanifests(commonrevs, clrevorder,
                fastpathlinkrev, mfs, fnodes, source,
                clstate['clrevtomanifestrev']):
            yield chunk

        mfdicts = None
        if self._ellipses and self._isshallow:
            mfdicts = [(self._repo.manifestlog[n].read(), lr)

            'mfs': mfs,
            'changedfiles': changedfiles,
            'clrevtomanifestrev': clrevtomanifestrev,
        }

        gen = self.group(revs, cl, True, lookupcl, units=_('changesets'),
                         clrevtolocalrev={})

        return state, gen

    def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
                          fnodes, source, clrevtolocalrev):
        """Returns an iterator of changegroup chunks containing manifests.

        `source` is unused here, but is used by extensions like remotefilelog to
        change what is sent based in pulls vs pushes, etc.
        """

                                              lookupfn)
                else:
                    revs = _sortnodesnormal(store, prunednodes,
                                            self._reorder)

                for x in self._packmanifests(dir, store, revs, lookupfn,
                                             clrevtolocalrev):
                    size += len(x)
                    yield x
        self._verbosenote(_('%8.i (manifests)\n') % size)
        yield self._manifestsend

                flinkrev = store.linkrev
                fnode = store.node
                revs = ((r, flinkrev(r)) for r in store)
                return dict((fnode(r), cln(lr))
                            for r, lr in revs if lr in clrevs)

        clrevtolocalrev = {}

        if self._isshallow:
            # In a shallow clone, the linknodes callback needs to also include
            # those file nodes that are in the manifests we sent but weren't
            # introduced by those manifests.

            # TODO have caller pass in appropriate function.
            def linknodes(flog, fname):
                for c in commonctxs:
                    try:
                        fnode = c.filenode(fname)
                        clrevtolocalrev[c.rev()] = flog.rev(fnode)
                    except error.ManifestLookupError:
                        pass
                links = normallinknodes(flog, fname)
                if len(links) != len(mfdicts):
                    for mf, lr in mfdicts:

            filerevlog = repo.file(fname)
            if not filerevlog:
                raise error.Abort(_("empty or missing file data for %s") %
                                  fname)

            clrevtolocalrev.clear()

            linkrevnodes = linknodes(filerevlog, fname)
            # Lookup for filenodes, we collected the linkrev nodes above in the
            # fastpath case and with lookupmf in the slowpath case.
            def lookupfilelog(x):
                return linkrevnodes[x]

                progress.update(i + 1, item=fname)
                h = _fileheader(fname)
                size = len(h)
                yield h
                for chunk in self.group(revs, filerevlog, False, lookupfilelog,
                                        clrevtolocalrev=clrevtolocalrev):
                    size += len(chunk)
                    yield chunk
                self._verbosenote(_('%8.i  %s\n') % (size, fname))
        progress.complete()

    def _revisiondeltanarrow(self, store, ischangelog, rev, linkrev, linknode,
                             clrevtolocalrev):
        linkparents = self._precomputedellipsis[linkrev]
        def local(clrev):
            """Turn a changelog revnum into a local revnum.

            The ellipsis dag is stored as revnums on the changelog,

            # nodes even after ellipsis-izing.
            walk = [clrev]
            while walk:
                p = walk[0]
                walk = walk[1:]
                if p in clrevtolocalrev:
                    return clrevtolocalrev[p]
                elif p in self._fullclnodes:
                    walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
                                    if pp != nullrev])
                elif p in self._precomputedellipsis:
                    walk.extend([pp for pp in self._precomputedellipsis[p]