comparison mercurial/changegroup.py @ 38903:68e490ed640e

changegroup: make some packer attributes private

These methods and attributes are low level and should not be
called or outside outside of instances. Indicate as such through
naming.

Differential Revision: https://phab.mercurial-scm.org/D4085

38902:4c99c6d1ef95

        if self._repo.ui.verbose and not self._repo.ui.debugflag:
            self._verbosenote = self._repo.ui.note
        else:
            self._verbosenote = lambda s: None

    def close(self):
        # Ellipses serving mode.
        getattr(self, 'clrev_to_localrev', {}).clear()
        if getattr(self, 'next_clrev_to_localrev', {}):
            self.clrev_to_localrev = self.next_clrev_to_localrev
            del self.next_clrev_to_localrev
        self.changelog_done = True

        return closechunk()

    def fileheader(self, fname):
        return chunkheader(len(fname)) + fname

    # Extracted both for clarity and for overriding in extensions.
    def _sortgroup(self, store, nodelist, lookup):
        """Sort nodes for change group and turn them into revnums."""

        # The one invariant we *know* holds is that the new (potentially
        # bogus) DAG shape will be valid if we order the nodes in the
        # order that they're introduced in dramatis personae by the
        # changelog, so what we do is we sort the non-changelog histories
        # by the order in which they are used by the changelog.
        if util.safehasattr(self, 'full_nodes') and self.clnode_to_rev:
            key = lambda n: self.clnode_to_rev[lookup(n)]
            return [store.rev(n) for n in sorted(nodelist, key=key)]

        # for generaldelta revlogs, we linearize the revs; this will both be
        # much quicker and generate a much smaller bundle
        if (store._generaldelta and self._reorder is None) or self._reorder:

        If units is not None, progress detail will be generated, units specifies
        the type of revlog that is touched (changelog, manifest, etc.).
        """
        # if we don't have any revisions touched by these changesets, bail
        if len(nodelist) == 0:
            yield self.close()
            return

        revs = self._sortgroup(store, nodelist, lookup)

        # add the parent of the first rev

        for r in pycompat.xrange(len(revs) - 1):
            if progress:
                progress.update(r + 1)
            prev, curr = revs[r], revs[r + 1]
            linknode = lookup(store.node(curr))
            for c in self.revchunk(store, curr, prev, linknode):
                yield c

        if progress:
            progress.complete()
        yield self.close()

    # filter any nodes that claim to be part of the known set
    def prune(self, store, missing, commonrevs):
        # TODO this violates storage abstraction for manifests.
        if isinstance(store, manifest.manifestrevlog):
            if not self._filematcher.visitdir(store._dir[:-1] or '.'):
                return []

        can be sent.
        """
        assert self.version == b'03'

        if dir:
            yield self.fileheader(dir)

        # TODO violates storage abstractions by assuming revlogs.
        dirlog = self._repo.manifestlog._revlog.dirlog(dir)
        for chunk in self.group(mfnodes, dirlog, lookuplinknode,
                                units=_('manifests')):

        mfl = repo.manifestlog
        # TODO violates storage abstraction.
        mfrevlog = mfl._revlog
        changedfiles = set()

        ellipsesmode = util.safehasattr(self, 'full_nodes')

        # Callback for the changelog, used to collect changed files and
        # manifest nodes.
        # Returns the linkrev node (identity in the changelog case).
        def lookupcl(x):

            if ellipsesmode:
                # Only update mfs if x is going to be sent. Otherwise we
                # end up with bogus linkrevs specified for manifests and
                # we skip some manifest nodes that we should otherwise
                # have sent.
                if (x in self.full_nodes
                    or cl.rev(x) in self.precomputed_ellipsis):
                    n = c[0]
                    # Record the first changeset introducing this manifest
                    # version.
                    mfs.setdefault(n, x)
                    # Set this narrow-specific dict so we have the lowest
                    # manifest revnum to look up for this cl revnum. (Part of
                    # mapping changelog ellipsis parents to manifest ellipsis
                    # parents)
                    self.next_clrev_to_localrev.setdefault(cl.rev(x),
                                                           mfrevlog.rev(n))
                # We can't trust the changed files list in the changeset if the
                # client requested a shallow clone.
                if self.is_shallow:
                    changedfiles.update(mfl[c[0]].read().keys())
                else:
                    changedfiles.update(c[3])
            else:

                fastpathlinkrev, mfs, fnodes, source):
            yield chunk

        if ellipsesmode:
            mfdicts = None
            if self.is_shallow:
                mfdicts = [(self._repo.manifestlog[n].read(), lr)
                           for (n, lr) in mfs.iteritems()]

        mfs.clear()
        clrevs = set(cl.rev(x) for x in clnodes)

        for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
                                        source):
            yield chunk

        yield self.close()

        if clnodes:
            repo.hook('outgoing', node=hex(clnodes[0]), source=source)

    def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,

        fn = (self._packtreemanifests if self._sendtreemanifests
              else self._packmanifests)
        size = 0
        while tmfnodes:
            dir, nodes = tmfnodes.popitem()
            prunednodes = self.prune(dirlog(dir), nodes, commonrevs)
            if not dir or prunednodes:
                for x in fn(dir, prunednodes, makelookupmflinknode(dir, nodes)):
                    size += len(x)
                    yield x
        self._verbosenote(_('%8.i (manifests)\n') % size)

    # The 'source' parameter is useful for extensions
    def generatefiles(self, changedfiles, linknodes, commonrevs, source):
        changedfiles = list(filter(self._filematcher, changedfiles))

        if getattr(self, 'is_shallow', False):
            # See comment in generate() for why this sadness is a thing.
            mfdicts = self._mfdicts
            del self._mfdicts
            # In a shallow clone, the linknodes callback needs to also include
            # those file nodes that are in the manifests we sent but weren't

            # TODO have caller pass in appropriate function.
            def linknodes(flog, fname):
                for c in commonctxs:
                    try:
                        fnode = c.filenode(fname)
                        self.clrev_to_localrev[c.rev()] = flog.rev(fnode)
                    except error.ManifestLookupError:
                        pass
                links = oldlinknodes(flog, fname)
                if len(links) != len(mfdicts):
                    for mf, lr in mfdicts:

            # Lookup for filenodes, we collected the linkrev nodes above in the
            # fastpath case and with lookupmf in the slowpath case.
            def lookupfilelog(x):
                return linkrevnodes[x]

            filenodes = self.prune(filerevlog, linkrevnodes, commonrevs)
            if filenodes:
                progress.update(i + 1, item=fname)
                h = self.fileheader(fname)
                size = len(h)
                yield h
                for chunk in self.group(filenodes, filerevlog, lookupfilelog):
                    size += len(chunk)
                    yield chunk
                self._verbosenote(_('%8.i  %s\n') % (size, fname))
        progress.complete()

    def deltaparent(self, store, rev, p1, p2, prev):
        if self._useprevdelta:
            if not store.candelta(prev, rev):
                raise error.ProgrammingError(
                    'cg1 should not be used in this case')
            return prev

        # Narrow ellipses mode.
        if util.safehasattr(self, 'full_nodes'):
            # TODO: send better deltas when in narrow mode.
            #
            # changegroup.group() loops over revisions to send,
            # including revisions we'll skip. What this means is that
            # `prev` will be a potentially useless delta base for all

        if base != nullrev and not store.candelta(base, rev):
            base = nullrev

        return base

    def revchunk(self, store, rev, prev, linknode):
        if util.safehasattr(self, 'full_nodes'):
            fn = self._revisiondeltanarrow
        else:
            fn = self._revisiondeltanormal

        delta = fn(store, rev, prev, linknode)

            yield x

    def _revisiondeltanormal(self, store, rev, prev, linknode):
        node = store.node(rev)
        p1, p2 = store.parentrevs(rev)
        base = self.deltaparent(store, rev, p1, p2, prev)

        prefix = ''
        if store.iscensored(base) or store.iscensored(rev):
            try:
                delta = store.revision(node, raw=True)

        )

    def _revisiondeltanarrow(self, store, rev, prev, linknode):
        # build up some mapping information that's useful later. See
        # the local() nested function below.
        if not self.changelog_done:
            self.clnode_to_rev[linknode] = rev
            linkrev = rev
            self.clrev_to_localrev[linkrev] = rev
        else:
            linkrev = self.clnode_to_rev[linknode]
            self.clrev_to_localrev[linkrev] = rev

        # This is a node to send in full, because the changeset it
        # corresponds to was a full changeset.
        if linknode in self.full_nodes:
            return self._revisiondeltanormal(store, rev, prev, linknode)

        # At this point, a node can either be one we should skip or an
        # ellipsis. If it's not an ellipsis, bail immediately.
        if linkrev not in self.precomputed_ellipsis:
            return

        linkparents = self.precomputed_ellipsis[linkrev]
        def local(clrev):
            """Turn a changelog revnum into a local revnum.

            The ellipsis dag is stored as revnums on the changelog,
            but when we're producing ellipsis entries for

            mappings as needed.
            """
            if clrev == nullrev:
                return nullrev

            if not self.changelog_done:
                # If we're doing the changelog, it's possible that we
                # have a parent that is already on the client, and we
                # need to store some extra mapping information so that
                # our contained ellipsis nodes will be able to resolve
                # their parents.
                if clrev not in self.clrev_to_localrev:
                    clnode = store.node(clrev)
                    self.clnode_to_rev[clnode] = clrev
                return clrev

            # Walk the ellipsis-ized changelog breadth-first looking for a
            # change that has been linked from the current revlog.
            #

            # nodes even after ellipsis-izing.
            walk = [clrev]
            while walk:
                p = walk[0]
                walk = walk[1:]
                if p in self.clrev_to_localrev:
                    return self.clrev_to_localrev[p]
                elif p in self.full_nodes:
                    walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
                                    if pp != nullrev])
                elif p in self.precomputed_ellipsis:
                    walk.extend([pp for pp in self.precomputed_ellipsis[p]
                                    if pp != nullrev])
                else:
                    # In this case, we've got an ellipsis with parents
                    # outside the current bundle (likely an
                    # incremental pull). We "know" that we can use the

    packer = getbundler(version, repo, filematcher=match)
    # Give the packer the list of nodes which should not be
    # ellipsis nodes. We store this rather than the set of nodes
    # that should be an ellipsis because for very large histories
    # we expect this to be significantly smaller.
    packer.full_nodes = relevant_nodes
    # Maps ellipsis revs to their roots at the changelog level.
    packer.precomputed_ellipsis = ellipsisroots
    # Maps CL revs to per-revlog revisions. Cleared in close() at
    # the end of each group.
    packer.clrev_to_localrev = {}
    packer.next_clrev_to_localrev = {}
    # Maps changelog nodes to changelog revs. Filled in once
    # during changelog stage and then left unmodified.
    packer.clnode_to_rev = {}
    packer.changelog_done = False
    # If true, informs the packer that it is serving shallow content and might
    # need to pack file contents not introduced by the changes being packed.
    packer.is_shallow = depth is not None

    return packer.generate(common, visitnodes, False, source)

38903:68e490ed640e

        if self._repo.ui.verbose and not self._repo.ui.debugflag:
            self._verbosenote = self._repo.ui.note
        else:
            self._verbosenote = lambda s: None

    def _close(self):
        # Ellipses serving mode.
        getattr(self, '_clrev_to_localrev', {}).clear()
        if getattr(self, '_next_clrev_to_localrev', {}):
            self._clrev_to_localrev = self._next_clrev_to_localrev
            del self._next_clrev_to_localrev
        self._changelog_done = True

        return closechunk()

    def _fileheader(self, fname):
        return chunkheader(len(fname)) + fname

    # Extracted both for clarity and for overriding in extensions.
    def _sortgroup(self, store, nodelist, lookup):
        """Sort nodes for change group and turn them into revnums."""

        # The one invariant we *know* holds is that the new (potentially
        # bogus) DAG shape will be valid if we order the nodes in the
        # order that they're introduced in dramatis personae by the
        # changelog, so what we do is we sort the non-changelog histories
        # by the order in which they are used by the changelog.
        if util.safehasattr(self, '_full_nodes') and self._clnode_to_rev:
            key = lambda n: self._clnode_to_rev[lookup(n)]
            return [store.rev(n) for n in sorted(nodelist, key=key)]

        # for generaldelta revlogs, we linearize the revs; this will both be
        # much quicker and generate a much smaller bundle
        if (store._generaldelta and self._reorder is None) or self._reorder:

        If units is not None, progress detail will be generated, units specifies
        the type of revlog that is touched (changelog, manifest, etc.).
        """
        # if we don't have any revisions touched by these changesets, bail
        if len(nodelist) == 0:
            yield self._close()
            return

        revs = self._sortgroup(store, nodelist, lookup)

        # add the parent of the first rev

        for r in pycompat.xrange(len(revs) - 1):
            if progress:
                progress.update(r + 1)
            prev, curr = revs[r], revs[r + 1]
            linknode = lookup(store.node(curr))
            for c in self._revchunk(store, curr, prev, linknode):
                yield c

        if progress:
            progress.complete()
        yield self._close()

    # filter any nodes that claim to be part of the known set
    def _prune(self, store, missing, commonrevs):
        # TODO this violates storage abstraction for manifests.
        if isinstance(store, manifest.manifestrevlog):
            if not self._filematcher.visitdir(store._dir[:-1] or '.'):
                return []

        can be sent.
        """
        assert self.version == b'03'

        if dir:
            yield self._fileheader(dir)

        # TODO violates storage abstractions by assuming revlogs.
        dirlog = self._repo.manifestlog._revlog.dirlog(dir)
        for chunk in self.group(mfnodes, dirlog, lookuplinknode,
                                units=_('manifests')):

        mfl = repo.manifestlog
        # TODO violates storage abstraction.
        mfrevlog = mfl._revlog
        changedfiles = set()

        ellipsesmode = util.safehasattr(self, '_full_nodes')

        # Callback for the changelog, used to collect changed files and
        # manifest nodes.
        # Returns the linkrev node (identity in the changelog case).
        def lookupcl(x):

            if ellipsesmode:
                # Only update mfs if x is going to be sent. Otherwise we
                # end up with bogus linkrevs specified for manifests and
                # we skip some manifest nodes that we should otherwise
                # have sent.
                if (x in self._full_nodes
                    or cl.rev(x) in self._precomputed_ellipsis):
                    n = c[0]
                    # Record the first changeset introducing this manifest
                    # version.
                    mfs.setdefault(n, x)
                    # Set this narrow-specific dict so we have the lowest
                    # manifest revnum to look up for this cl revnum. (Part of
                    # mapping changelog ellipsis parents to manifest ellipsis
                    # parents)
                    self._next_clrev_to_localrev.setdefault(cl.rev(x),
                                                            mfrevlog.rev(n))
                # We can't trust the changed files list in the changeset if the
                # client requested a shallow clone.
                if self._is_shallow:
                    changedfiles.update(mfl[c[0]].read().keys())
                else:
                    changedfiles.update(c[3])
            else:

                fastpathlinkrev, mfs, fnodes, source):
            yield chunk

        if ellipsesmode:
            mfdicts = None
            if self._is_shallow:
                mfdicts = [(self._repo.manifestlog[n].read(), lr)
                           for (n, lr) in mfs.iteritems()]

        mfs.clear()
        clrevs = set(cl.rev(x) for x in clnodes)

        for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
                                        source):
            yield chunk

        yield self._close()

        if clnodes:
            repo.hook('outgoing', node=hex(clnodes[0]), source=source)

    def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,

        fn = (self._packtreemanifests if self._sendtreemanifests
              else self._packmanifests)
        size = 0
        while tmfnodes:
            dir, nodes = tmfnodes.popitem()
            prunednodes = self._prune(dirlog(dir), nodes, commonrevs)
            if not dir or prunednodes:
                for x in fn(dir, prunednodes, makelookupmflinknode(dir, nodes)):
                    size += len(x)
                    yield x
        self._verbosenote(_('%8.i (manifests)\n') % size)

    # The 'source' parameter is useful for extensions
    def generatefiles(self, changedfiles, linknodes, commonrevs, source):
        changedfiles = list(filter(self._filematcher, changedfiles))

        if getattr(self, '_is_shallow', False):
            # See comment in generate() for why this sadness is a thing.
            mfdicts = self._mfdicts
            del self._mfdicts
            # In a shallow clone, the linknodes callback needs to also include
            # those file nodes that are in the manifests we sent but weren't

            # TODO have caller pass in appropriate function.
            def linknodes(flog, fname):
                for c in commonctxs:
                    try:
                        fnode = c.filenode(fname)
                        self._clrev_to_localrev[c.rev()] = flog.rev(fnode)
                    except error.ManifestLookupError:
                        pass
                links = oldlinknodes(flog, fname)
                if len(links) != len(mfdicts):
                    for mf, lr in mfdicts:

            # Lookup for filenodes, we collected the linkrev nodes above in the
            # fastpath case and with lookupmf in the slowpath case.
            def lookupfilelog(x):
                return linkrevnodes[x]

            filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
            if filenodes:
                progress.update(i + 1, item=fname)
                h = self._fileheader(fname)
                size = len(h)
                yield h
                for chunk in self.group(filenodes, filerevlog, lookupfilelog):
                    size += len(chunk)
                    yield chunk
                self._verbosenote(_('%8.i  %s\n') % (size, fname))
        progress.complete()

    def _deltaparent(self, store, rev, p1, p2, prev):
        if self._useprevdelta:
            if not store.candelta(prev, rev):
                raise error.ProgrammingError(
                    'cg1 should not be used in this case')
            return prev

        # Narrow ellipses mode.
        if util.safehasattr(self, '_full_nodes'):
            # TODO: send better deltas when in narrow mode.
            #
            # changegroup.group() loops over revisions to send,
            # including revisions we'll skip. What this means is that
            # `prev` will be a potentially useless delta base for all

        if base != nullrev and not store.candelta(base, rev):
            base = nullrev

        return base

    def _revchunk(self, store, rev, prev, linknode):
        if util.safehasattr(self, '_full_nodes'):
            fn = self._revisiondeltanarrow
        else:
            fn = self._revisiondeltanormal

        delta = fn(store, rev, prev, linknode)

            yield x

    def _revisiondeltanormal(self, store, rev, prev, linknode):
        node = store.node(rev)
        p1, p2 = store.parentrevs(rev)
        base = self._deltaparent(store, rev, p1, p2, prev)

        prefix = ''
        if store.iscensored(base) or store.iscensored(rev):
            try:
                delta = store.revision(node, raw=True)

        )

    def _revisiondeltanarrow(self, store, rev, prev, linknode):
        # build up some mapping information that's useful later. See
        # the local() nested function below.
        if not self._changelog_done:
            self._clnode_to_rev[linknode] = rev
            linkrev = rev
            self._clrev_to_localrev[linkrev] = rev
        else:
            linkrev = self._clnode_to_rev[linknode]
            self._clrev_to_localrev[linkrev] = rev

        # This is a node to send in full, because the changeset it
        # corresponds to was a full changeset.
        if linknode in self._full_nodes:
            return self._revisiondeltanormal(store, rev, prev, linknode)

        # At this point, a node can either be one we should skip or an
        # ellipsis. If it's not an ellipsis, bail immediately.
        if linkrev not in self._precomputed_ellipsis:
            return

        linkparents = self._precomputed_ellipsis[linkrev]
        def local(clrev):
            """Turn a changelog revnum into a local revnum.

            The ellipsis dag is stored as revnums on the changelog,
            but when we're producing ellipsis entries for

            mappings as needed.
            """
            if clrev == nullrev:
                return nullrev

            if not self._changelog_done:
                # If we're doing the changelog, it's possible that we
                # have a parent that is already on the client, and we
                # need to store some extra mapping information so that
                # our contained ellipsis nodes will be able to resolve
                # their parents.
                if clrev not in self._clrev_to_localrev:
                    clnode = store.node(clrev)
                    self._clnode_to_rev[clnode] = clrev
                return clrev

            # Walk the ellipsis-ized changelog breadth-first looking for a
            # change that has been linked from the current revlog.
            #

            # nodes even after ellipsis-izing.
            walk = [clrev]
            while walk:
                p = walk[0]
                walk = walk[1:]
                if p in self._clrev_to_localrev:
                    return self._clrev_to_localrev[p]
                elif p in self._full_nodes:
                    walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
                                    if pp != nullrev])
                elif p in self._precomputed_ellipsis:
                    walk.extend([pp for pp in self._precomputed_ellipsis[p]
                                    if pp != nullrev])
                else:
                    # In this case, we've got an ellipsis with parents
                    # outside the current bundle (likely an
                    # incremental pull). We "know" that we can use the

    packer = getbundler(version, repo, filematcher=match)
    # Give the packer the list of nodes which should not be
    # ellipsis nodes. We store this rather than the set of nodes
    # that should be an ellipsis because for very large histories
    # we expect this to be significantly smaller.
    packer._full_nodes = relevant_nodes
    # Maps ellipsis revs to their roots at the changelog level.
    packer._precomputed_ellipsis = ellipsisroots
    # Maps CL revs to per-revlog revisions. Cleared in close() at
    # the end of each group.
    packer._clrev_to_localrev = {}
    packer._next_clrev_to_localrev = {}
    # Maps changelog nodes to changelog revs. Filled in once
    # during changelog stage and then left unmodified.
    packer._clnode_to_rev = {}
    packer._changelog_done = False
    # If true, informs the packer that it is serving shallow content and might
    # need to pack file contents not introduced by the changes being packed.
    packer._is_shallow = depth is not None

    return packer.generate(common, visitnodes, False, source)