comparison mercurial/localrepo.py @ 16420:4df2d335b8f6

changegroup: reduce progress overhead

16419:784606ae0029

        # filter any nodes that claim to be part of the known set
        def prune(revlog, missing):
            return [n for n in missing
                    if revlog.linkrev(revlog.rev(n)) not in commonrevs]

        def lookup(revlog, x):
            if revlog == cl:
                c = cl.read(x)
                changedfiles.update(c[3])
                mfs.setdefault(c[0], x)
                count[0] += 1
                self.ui.progress(_('bundling'), count[0],
                                 unit=_('changesets'), total=len(csets))
                return x
            elif revlog == mf:
                clnode = mfs[x]
                mdata = mf.readfast(x)
                for f in mdata:
                    if f in changedfiles:
                        fnodes.setdefault(f, {}).setdefault(mdata[f], clnode)
                count[0] += 1
                self.ui.progress(_('bundling'), count[0],
                                 unit=_('manifests'), total=len(mfs))
                return mfs[x]
            else:
                self.ui.progress(
                    _('bundling'), count[0], item=fstate[0],
                    unit=_('files'), total=len(changedfiles))
                return fstate[1][x]

        bundler = changegroup.bundle10(lookup)
        reorder = self.ui.config('bundle', 'reorder', 'auto')
        if reorder == 'auto':

        def gengroup():
            # Create a changenode group generator that will call our functions
            # back to lookup the owning changenode and collect information.
            for chunk in cl.group(csets, bundler, reorder=reorder):
                yield chunk
            self.ui.progress(_('bundling'), None)

            # Create a generator for the manifestnodes that calls our lookup
            # and data collection functions back.
            count[0] = 0
            for chunk in mf.group(prune(mf, mfs), bundler, reorder=reorder):
                yield chunk
            self.ui.progress(_('bundling'), None)

            mfs.clear()

            # Go through all our files in order sorted by name.
            count[0] = 0

                    for chunk in filerevlog.group(nodelist, bundler, reorder):
                        yield chunk

            # Signal that no more groups are left.
            yield bundler.close()
            self.ui.progress(_('bundling'), None)

            if csets:
                self.hook('outgoing', node=hex(csets[0]), source=source)

        return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN')

        revset = set([cl.rev(n) for n in nodes])

        def gennodelst(log):
            return [log.node(r) for r in log if log.linkrev(r) in revset]

        def lookup(revlog, x):
            if revlog == cl:
                c = cl.read(x)
                changedfiles.update(c[3])
                mfs.setdefault(c[0], x)
                count[0] += 1
                self.ui.progress(_('bundling'), count[0],
                                 unit=_('changesets'), total=len(nodes))
                return x
            elif revlog == mf:
                count[0] += 1
                self.ui.progress(_('bundling'), count[0],
                                 unit=_('manifests'), total=len(mfs))
                return cl.node(revlog.linkrev(revlog.rev(x)))
            else:
                self.ui.progress(
                    _('bundling'), count[0], item=fstate[0],
                    total=len(changedfiles), unit=_('files'))
                return cl.node(revlog.linkrev(revlog.rev(x)))

        bundler = changegroup.bundle10(lookup)
        reorder = self.ui.config('bundle', 'reorder', 'auto')
        if reorder == 'auto':

            '''yield a sequence of changegroup chunks (strings)'''
            # construct a list of all changed files

            for chunk in cl.group(nodes, bundler, reorder=reorder):
                yield chunk
            self.ui.progress(_('bundling'), None)

            count[0] = 0
            for chunk in mf.group(gennodelst(mf), bundler, reorder=reorder):
                yield chunk
            self.ui.progress(_('bundling'), None)

            count[0] = 0
            for fname in sorted(changedfiles):
                filerevlog = self.file(fname)
                if not len(filerevlog):

                    count[0] += 1
                    yield bundler.fileheader(fname)
                    for chunk in filerevlog.group(nodelist, bundler, reorder):
                        yield chunk
            yield bundler.close()
            self.ui.progress(_('bundling'), None)

            if nodes:
                self.hook('outgoing', node=hex(nodes[0]), source=source)

        return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN')

16420:4df2d335b8f6

        # filter any nodes that claim to be part of the known set
        def prune(revlog, missing):
            return [n for n in missing
                    if revlog.linkrev(revlog.rev(n)) not in commonrevs]

        progress = self.ui.progress
        _bundling = _('bundling')
        _changesets = _('changesets')
        _manifests = _('manifests')
        _files = _('files')

        def lookup(revlog, x):
            if revlog == cl:
                c = cl.read(x)
                changedfiles.update(c[3])
                mfs.setdefault(c[0], x)
                count[0] += 1
                progress(_bundling, count[0],
                         unit=_changesets, total=len(csets))
                return x
            elif revlog == mf:
                clnode = mfs[x]
                mdata = mf.readfast(x)
                for f in mdata:
                    if f in changedfiles:
                        fnodes.setdefault(f, {}).setdefault(mdata[f], clnode)
                count[0] += 1
                progress(_bundling, count[0],
                         unit=_manifests, total=len(mfs))
                return mfs[x]
            else:
                progress(_bundling, count[0], item=fstate[0],
                         unit=_files, total=len(changedfiles))
                return fstate[1][x]

        bundler = changegroup.bundle10(lookup)
        reorder = self.ui.config('bundle', 'reorder', 'auto')
        if reorder == 'auto':

        def gengroup():
            # Create a changenode group generator that will call our functions
            # back to lookup the owning changenode and collect information.
            for chunk in cl.group(csets, bundler, reorder=reorder):
                yield chunk
            progress(_bundling, None)

            # Create a generator for the manifestnodes that calls our lookup
            # and data collection functions back.
            count[0] = 0
            for chunk in mf.group(prune(mf, mfs), bundler, reorder=reorder):
                yield chunk
            progress(_bundling, None)

            mfs.clear()

            # Go through all our files in order sorted by name.
            count[0] = 0

                    for chunk in filerevlog.group(nodelist, bundler, reorder):
                        yield chunk

            # Signal that no more groups are left.
            yield bundler.close()
            progress(_bundling, None)

            if csets:
                self.hook('outgoing', node=hex(csets[0]), source=source)

        return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN')

        revset = set([cl.rev(n) for n in nodes])

        def gennodelst(log):
            return [log.node(r) for r in log if log.linkrev(r) in revset]
        progress = self.ui.progress
        _bundling = _('bundling')
        _changesets = _('changesets')
        _manifests = _('manifests')
        _files = _('files')

        def lookup(revlog, x):
            if revlog == cl:
                c = cl.read(x)
                changedfiles.update(c[3])
                mfs.setdefault(c[0], x)
                count[0] += 1
                progress(_bundling, count[0],
                         unit=_changesets, total=len(nodes))
                return x
            elif revlog == mf:
                count[0] += 1
                progress(_bundling, count[0],
                         unit=_manifests, total=len(mfs))
                return cl.node(revlog.linkrev(revlog.rev(x)))
            else:
                progress(_bundling, count[0], item=fstate[0],
                    total=len(changedfiles), unit=_files)
                return cl.node(revlog.linkrev(revlog.rev(x)))

        bundler = changegroup.bundle10(lookup)
        reorder = self.ui.config('bundle', 'reorder', 'auto')
        if reorder == 'auto':

            '''yield a sequence of changegroup chunks (strings)'''
            # construct a list of all changed files

            for chunk in cl.group(nodes, bundler, reorder=reorder):
                yield chunk
            progress(_bundling, None)

            count[0] = 0
            for chunk in mf.group(gennodelst(mf), bundler, reorder=reorder):
                yield chunk
            progress(_bundling, None)

            count[0] = 0
            for fname in sorted(changedfiles):
                filerevlog = self.file(fname)
                if not len(filerevlog):

                    count[0] += 1
                    yield bundler.fileheader(fname)
                    for chunk in filerevlog.group(nodelist, bundler, reorder):
                        yield chunk
            yield bundler.close()
            progress(_bundling, None)

            if nodes:
                self.hook('outgoing', node=hex(nodes[0]), source=source)

        return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN')