comparison mercurial/changegroup.py @ 38891:205c98e2f1ba

changegroup: rename "revlog" variables

"revlog" shadows the module import. But more importantly, changegroup
generation should be storage agnostic and not assume the existence
of revlogs. Let's rename the thing providing revision storage to
"store" to reflect this ideal property.

Differential Revision: https://phab.mercurial-scm.org/D4073

38890:d706c77449f9

    def fileheader(self, fname):
        return chunkheader(len(fname)) + fname

    # Extracted both for clarity and for overriding in extensions.
    def _sortgroup(self, revlog, nodelist, lookup):
        """Sort nodes for change group and turn them into revnums."""
        # Ellipses serving mode.
        #
        # In a perfect world, we'd generate better ellipsis-ified graphs
        # for non-changelog revlogs. In practice, we haven't started doing

        # order that they're introduced in dramatis personae by the
        # changelog, so what we do is we sort the non-changelog histories
        # by the order in which they are used by the changelog.
        if util.safehasattr(self, 'full_nodes') and self.clnode_to_rev:
            key = lambda n: self.clnode_to_rev[lookup(n)]
            return [revlog.rev(n) for n in sorted(nodelist, key=key)]

        # for generaldelta revlogs, we linearize the revs; this will both be
        # much quicker and generate a much smaller bundle
        if (revlog._generaldelta and self._reorder is None) or self._reorder:
            dag = dagutil.revlogdag(revlog)
            return dag.linearize(set(revlog.rev(n) for n in nodelist))
        else:
            return sorted([revlog.rev(n) for n in nodelist])

    def group(self, nodelist, revlog, lookup, units=None):
        """Calculate a delta group, yielding a sequence of changegroup chunks
        (strings).

        Given a list of changeset revs, return a set of deltas and
        metadata corresponding to nodes. The first delta is

        # if we don't have any revisions touched by these changesets, bail
        if len(nodelist) == 0:
            yield self.close()
            return

        revs = self._sortgroup(revlog, nodelist, lookup)

        # add the parent of the first rev
        p = revlog.parentrevs(revs[0])[0]
        revs.insert(0, p)

        # build deltas
        progress = None
        if units is not None:

                                                  total=(len(revs) - 1))
        for r in pycompat.xrange(len(revs) - 1):
            if progress:
                progress.update(r + 1)
            prev, curr = revs[r], revs[r + 1]
            linknode = lookup(revlog.node(curr))
            for c in self.revchunk(revlog, curr, prev, linknode):
                yield c

        if progress:
            progress.complete()
        yield self.close()

    # filter any nodes that claim to be part of the known set
    def prune(self, revlog, missing, commonrevs):
        # TODO this violates storage abstraction for manifests.
        if isinstance(revlog, manifest.manifestrevlog):
            if not self._filematcher.visitdir(revlog._dir[:-1] or '.'):
                return []

        rr, rl = revlog.rev, revlog.linkrev
        return [n for n in missing if rl(rr(n)) not in commonrevs]

    def _packmanifests(self, dir, mfnodes, lookuplinknode):
        """Pack flat manifests into a changegroup stream."""
        assert not dir

                    size += len(chunk)
                    yield chunk
                self._verbosenote(_('%8.i  %s\n') % (size, fname))
        progress.complete()

    def deltaparent(self, revlog, rev, p1, p2, prev):
        if not revlog.candelta(prev, rev):
            raise error.ProgrammingError('cg1 should not be used in this case')
        return prev

    def revchunk(self, revlog, rev, prev, linknode):
        if util.safehasattr(self, 'full_nodes'):
            fn = self._revchunknarrow
        else:
            fn = self._revchunknormal

        return fn(revlog, rev, prev, linknode)

    def _revchunknormal(self, revlog, rev, prev, linknode):
        node = revlog.node(rev)
        p1, p2 = revlog.parentrevs(rev)
        base = self.deltaparent(revlog, rev, p1, p2, prev)

        prefix = ''
        if revlog.iscensored(base) or revlog.iscensored(rev):
            try:
                delta = revlog.revision(node, raw=True)
            except error.CensoredNodeError as e:
                delta = e.tombstone
            if base == nullrev:
                prefix = mdiff.trivialdiffheader(len(delta))
            else:
                baselen = revlog.rawsize(base)
                prefix = mdiff.replacediffheader(baselen, len(delta))
        elif base == nullrev:
            delta = revlog.revision(node, raw=True)
            prefix = mdiff.trivialdiffheader(len(delta))
        else:
            delta = revlog.revdiff(base, rev)
        p1n, p2n = revlog.parents(node)
        basenode = revlog.node(base)
        flags = revlog.flags(rev)
        meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags)
        meta += prefix
        l = len(meta) + len(delta)
        yield chunkheader(l)
        yield meta
        yield delta

    def _revchunknarrow(self, revlog, rev, prev, linknode):
        # build up some mapping information that's useful later. See
        # the local() nested function below.
        if not self.changelog_done:
            self.clnode_to_rev[linknode] = rev
            linkrev = rev

            self.clrev_to_localrev[linkrev] = rev

        # This is a node to send in full, because the changeset it
        # corresponds to was a full changeset.
        if linknode in self.full_nodes:
            for x in self._revchunknormal(revlog, rev, prev, linknode):
                yield x
            return

        # At this point, a node can either be one we should skip or an
        # ellipsis. If it's not an ellipsis, bail immediately.

                # have a parent that is already on the client, and we
                # need to store some extra mapping information so that
                # our contained ellipsis nodes will be able to resolve
                # their parents.
                if clrev not in self.clrev_to_localrev:
                    clnode = revlog.node(clrev)
                    self.clnode_to_rev[clnode] = clrev
                return clrev

            # Walk the ellipsis-ized changelog breadth-first looking for a
            # change that has been linked from the current revlog.

                    # thorough) cases in our testsuite, but I would be
                    # somewhat unsurprised to find a case in the wild
                    # where this breaks down a bit. That said, I don't
                    # know if it would hurt anything.
                    for i in pycompat.xrange(rev, 0, -1):
                        if revlog.linkrev(i) == clrev:
                            return i
                    # We failed to resolve a parent for this node, so
                    # we crash the changegroup construction.
                    raise error.Abort(
                        'unable to resolve parent while packing %r %r'
                        ' for changeset %r' % (revlog.indexfile, rev, clrev))

            return nullrev

        if not linkparents or (
            revlog.parentrevs(rev) == (nullrev, nullrev)):
            p1, p2 = nullrev, nullrev
        elif len(linkparents) == 1:
            p1, = sorted(local(p) for p in linkparents)
            p2 = nullrev
        else:
            p1, p2 = sorted(local(p) for p in linkparents)
        n = revlog.node(rev)

        yield ellipsisdata(
            self, rev, revlog, p1, p2, revlog.revision(n), linknode)

    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
        # do nothing with basenode, it is implicitly the previous one in HG10
        # do nothing with flags, it is implicitly 0 for cg1 and cg2
        return struct.pack(self.deltaheader, node, p1n, p2n, linknode)

            # Since generaldelta is directly supported by cg2, reordering
            # generally doesn't help, so we disable it by default (treating
            # bundle.reorder=auto just like bundle.reorder=False).
            self._reorder = False

    def deltaparent(self, revlog, rev, p1, p2, prev):
        # Narrow ellipses mode.
        if util.safehasattr(self, 'full_nodes'):
            # TODO: send better deltas when in narrow mode.
            #
            # changegroup.group() loops over revisions to send,

            # significantly smarter about delta bases. This is
            # slightly tricky because this same code has to work for
            # all revlogs, and we don't have the linkrev/linknode here.
            return p1

        dp = revlog.deltaparent(rev)
        if dp == nullrev and revlog.storedeltachains:
            # Avoid sending full revisions when delta parent is null. Pick prev
            # in that case. It's tempting to pick p1 in this case, as p1 will
            # be smaller in the common case. However, computing a delta against
            # p1 may require resolving the raw text of p1, which could be
            # expensive. The revlog caches should have prev cached, meaning

        elif dp not in (p1, p2, prev):
            # Pick prev when we can't be sure remote has the base revision.
            return prev
        else:
            base = dp
        if base != nullrev and not revlog.candelta(base, rev):
            base = nullrev
        return base

    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
        # Do nothing with flags, it is implicitly 0 in cg1 and cg2

38891:205c98e2f1ba

    def fileheader(self, fname):
        return chunkheader(len(fname)) + fname

    # Extracted both for clarity and for overriding in extensions.
    def _sortgroup(self, store, nodelist, lookup):
        """Sort nodes for change group and turn them into revnums."""
        # Ellipses serving mode.
        #
        # In a perfect world, we'd generate better ellipsis-ified graphs
        # for non-changelog revlogs. In practice, we haven't started doing

        # order that they're introduced in dramatis personae by the
        # changelog, so what we do is we sort the non-changelog histories
        # by the order in which they are used by the changelog.
        if util.safehasattr(self, 'full_nodes') and self.clnode_to_rev:
            key = lambda n: self.clnode_to_rev[lookup(n)]
            return [store.rev(n) for n in sorted(nodelist, key=key)]

        # for generaldelta revlogs, we linearize the revs; this will both be
        # much quicker and generate a much smaller bundle
        if (store._generaldelta and self._reorder is None) or self._reorder:
            dag = dagutil.revlogdag(store)
            return dag.linearize(set(store.rev(n) for n in nodelist))
        else:
            return sorted([store.rev(n) for n in nodelist])

    def group(self, nodelist, store, lookup, units=None):
        """Calculate a delta group, yielding a sequence of changegroup chunks
        (strings).

        Given a list of changeset revs, return a set of deltas and
        metadata corresponding to nodes. The first delta is

        # if we don't have any revisions touched by these changesets, bail
        if len(nodelist) == 0:
            yield self.close()
            return

        revs = self._sortgroup(store, nodelist, lookup)

        # add the parent of the first rev
        p = store.parentrevs(revs[0])[0]
        revs.insert(0, p)

        # build deltas
        progress = None
        if units is not None:

                                                  total=(len(revs) - 1))
        for r in pycompat.xrange(len(revs) - 1):
            if progress:
                progress.update(r + 1)
            prev, curr = revs[r], revs[r + 1]
            linknode = lookup(store.node(curr))
            for c in self.revchunk(store, curr, prev, linknode):
                yield c

        if progress:
            progress.complete()
        yield self.close()

    # filter any nodes that claim to be part of the known set
    def prune(self, store, missing, commonrevs):
        # TODO this violates storage abstraction for manifests.
        if isinstance(store, manifest.manifestrevlog):
            if not self._filematcher.visitdir(store._dir[:-1] or '.'):
                return []

        rr, rl = store.rev, store.linkrev
        return [n for n in missing if rl(rr(n)) not in commonrevs]

    def _packmanifests(self, dir, mfnodes, lookuplinknode):
        """Pack flat manifests into a changegroup stream."""
        assert not dir

                    size += len(chunk)
                    yield chunk
                self._verbosenote(_('%8.i  %s\n') % (size, fname))
        progress.complete()

    def deltaparent(self, store, rev, p1, p2, prev):
        if not store.candelta(prev, rev):
            raise error.ProgrammingError('cg1 should not be used in this case')
        return prev

    def revchunk(self, store, rev, prev, linknode):
        if util.safehasattr(self, 'full_nodes'):
            fn = self._revchunknarrow
        else:
            fn = self._revchunknormal

        return fn(store, rev, prev, linknode)

    def _revchunknormal(self, store, rev, prev, linknode):
        node = store.node(rev)
        p1, p2 = store.parentrevs(rev)
        base = self.deltaparent(store, rev, p1, p2, prev)

        prefix = ''
        if store.iscensored(base) or store.iscensored(rev):
            try:
                delta = store.revision(node, raw=True)
            except error.CensoredNodeError as e:
                delta = e.tombstone
            if base == nullrev:
                prefix = mdiff.trivialdiffheader(len(delta))
            else:
                baselen = store.rawsize(base)
                prefix = mdiff.replacediffheader(baselen, len(delta))
        elif base == nullrev:
            delta = store.revision(node, raw=True)
            prefix = mdiff.trivialdiffheader(len(delta))
        else:
            delta = store.revdiff(base, rev)
        p1n, p2n = store.parents(node)
        basenode = store.node(base)
        flags = store.flags(rev)
        meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags)
        meta += prefix
        l = len(meta) + len(delta)
        yield chunkheader(l)
        yield meta
        yield delta

    def _revchunknarrow(self, store, rev, prev, linknode):
        # build up some mapping information that's useful later. See
        # the local() nested function below.
        if not self.changelog_done:
            self.clnode_to_rev[linknode] = rev
            linkrev = rev

            self.clrev_to_localrev[linkrev] = rev

        # This is a node to send in full, because the changeset it
        # corresponds to was a full changeset.
        if linknode in self.full_nodes:
            for x in self._revchunknormal(store, rev, prev, linknode):
                yield x
            return

        # At this point, a node can either be one we should skip or an
        # ellipsis. If it's not an ellipsis, bail immediately.

                # have a parent that is already on the client, and we
                # need to store some extra mapping information so that
                # our contained ellipsis nodes will be able to resolve
                # their parents.
                if clrev not in self.clrev_to_localrev:
                    clnode = store.node(clrev)
                    self.clnode_to_rev[clnode] = clrev
                return clrev

            # Walk the ellipsis-ized changelog breadth-first looking for a
            # change that has been linked from the current revlog.

                    # thorough) cases in our testsuite, but I would be
                    # somewhat unsurprised to find a case in the wild
                    # where this breaks down a bit. That said, I don't
                    # know if it would hurt anything.
                    for i in pycompat.xrange(rev, 0, -1):
                        if store.linkrev(i) == clrev:
                            return i
                    # We failed to resolve a parent for this node, so
                    # we crash the changegroup construction.
                    raise error.Abort(
                        'unable to resolve parent while packing %r %r'
                        ' for changeset %r' % (store.indexfile, rev, clrev))

            return nullrev

        if not linkparents or (
            store.parentrevs(rev) == (nullrev, nullrev)):
            p1, p2 = nullrev, nullrev
        elif len(linkparents) == 1:
            p1, = sorted(local(p) for p in linkparents)
            p2 = nullrev
        else:
            p1, p2 = sorted(local(p) for p in linkparents)
        n = store.node(rev)

        yield ellipsisdata(
            self, rev, store, p1, p2, store.revision(n), linknode)

    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
        # do nothing with basenode, it is implicitly the previous one in HG10
        # do nothing with flags, it is implicitly 0 for cg1 and cg2
        return struct.pack(self.deltaheader, node, p1n, p2n, linknode)

            # Since generaldelta is directly supported by cg2, reordering
            # generally doesn't help, so we disable it by default (treating
            # bundle.reorder=auto just like bundle.reorder=False).
            self._reorder = False

    def deltaparent(self, store, rev, p1, p2, prev):
        # Narrow ellipses mode.
        if util.safehasattr(self, 'full_nodes'):
            # TODO: send better deltas when in narrow mode.
            #
            # changegroup.group() loops over revisions to send,

            # significantly smarter about delta bases. This is
            # slightly tricky because this same code has to work for
            # all revlogs, and we don't have the linkrev/linknode here.
            return p1

        dp = store.deltaparent(rev)
        if dp == nullrev and store.storedeltachains:
            # Avoid sending full revisions when delta parent is null. Pick prev
            # in that case. It's tempting to pick p1 in this case, as p1 will
            # be smaller in the common case. However, computing a delta against
            # p1 may require resolving the raw text of p1, which could be
            # expensive. The revlog caches should have prev cached, meaning

        elif dp not in (p1, p2, prev):
            # Pick prev when we can't be sure remote has the base revision.
            return prev
        else:
            base = dp
        if base != nullrev and not store.candelta(base, rev):
            base = nullrev
        return base

    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
        # Do nothing with flags, it is implicitly 0 in cg1 and cg2