comparison mercurial/changegroup.py @ 27237:c08814b48ae5

changegroup: avoid iterating the whole manifest

The old code gathered the list of all files that changed anywhere in
history and then gathered changed file nodes by walking the entirety
of each manifest to be sent in order to gather changed file
nodes. That's going to be unfortunate for narrowhg, and it's already
inefficient for medium-to-large repositories.

Timings for bundle --all on my hg repo, tested with hgperf:
Before:
! wall 23.442445 comb 23.440000 user 23.250000 sys 0.190000 (best of 3)

After:
! wall 20.272187 comb 20.270000 user 20.190000 sys 0.080000 (best of 3)

27236:b0d90fef16b6

        ml = repo.manifest

        clrevorder = {}
        mfs = {} # needed manifests
        fnodes = {} # needed file nodes
        changedfiles = set()

        # Callback for the changelog, used to collect changed files and manifest
        # nodes.
        # Returns the linkrev node (identity in the changelog case).
        def lookupcl(x):
            c = cl.read(x)
            clrevorder[x] = len(clrevorder)
            changedfiles.update(c[3])
            # record the first changeset introducing this manifest version
            mfs.setdefault(c[0], x)
            return x

        self._verbosenote(_('uncompressed size of bundle content:\n'))
        size = 0
        for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):

            after manifests have been sent to the client.
            """
            clnode = mfs[x]
            if not fastpathlinkrev:
                mdata = ml.readfast(x)
                for f, n in mdata.iteritems():
                    if f in changedfiles:
                        # record the first changeset introducing this filelog
                        # version
                        fclnodes = fnodes.setdefault(f, {})
                        fclnode = fclnodes.setdefault(n, clnode)
                        if clrevorder[clnode] < clrevorder[fclnode]:

                        if linkrev in clrevs:
                            yield filerevlog.node(r), cl.node(linkrev)
                return dict(genfilenodes())
            return fnodes.get(fname, {})

        for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
                                        source):
            yield chunk

        yield self.close()

27237:c08814b48ae5

        ml = repo.manifest

        clrevorder = {}
        mfs = {} # needed manifests
        fnodes = {} # needed file nodes
        # maps manifest node id -> set(changed files)
        mfchangedfiles = {}

        # Callback for the changelog, used to collect changed files and manifest
        # nodes.
        # Returns the linkrev node (identity in the changelog case).
        def lookupcl(x):
            c = cl.read(x)
            clrevorder[x] = len(clrevorder)
            n = c[0]
            # record the first changeset introducing this manifest version
            mfs.setdefault(n, x)
            # Record a complete list of potentially-changed files in
            # this manifest.
            mfchangedfiles.setdefault(n, set()).update(c[3])
            return x

        self._verbosenote(_('uncompressed size of bundle content:\n'))
        size = 0
        for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):

            after manifests have been sent to the client.
            """
            clnode = mfs[x]
            if not fastpathlinkrev:
                mdata = ml.readfast(x)
                for f in mfchangedfiles[x]:
                    if True:
                        try:
                            n = mdata[f]
                        except KeyError:
                            continue
                        # record the first changeset introducing this filelog
                        # version
                        fclnodes = fnodes.setdefault(f, {})
                        fclnode = fclnodes.setdefault(n, clnode)
                        if clrevorder[clnode] < clrevorder[fclnode]:

                        if linkrev in clrevs:
                            yield filerevlog.node(r), cl.node(linkrev)
                return dict(genfilenodes())
            return fnodes.get(fname, {})

        changedfiles = set()
        for x in mfchangedfiles.itervalues():
            changedfiles.update(x)
        for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
                                        source):
            yield chunk

        yield self.close()