comparison mercurial/localrepo.py @ 13706:7beb9834d185

changegroupsubset: minor cleanups

 - move some variable declarations
 - drop some excessive comments
 - use standard variable naming

13705:73cfb7a5aa56

        Another wrinkle is doing the reverse, figuring out which changeset in
        the changegroup a particular filenode or manifestnode belongs to.
        """

        # Set up some initial variables
        # Make it easy to refer to self.changelog
        cl = self.changelog
        # Compute the list of changesets in this changegroup.
        # Some bases may turn out to be superfluous, and some heads may be
        # too.  nodesbetween will return the minimal set of bases and heads
        # necessary to re-create the changegroup.
        if not bases:
            bases = [nullid]
        msng_cl_lst, bases, heads = cl.nodesbetween(bases, heads)

        # can we go through the fast path ?
        heads.sort()
        allheads = self.heads()
        allheads.sort()
        if heads == allheads:
            return self._changegroup(msng_cl_lst, source)

        # slow path
        self.hook('preoutgoing', throw=True, source=source)

        self.changegroupinfo(msng_cl_lst, source)

        # We assume that all ancestors of bases are known
        commonrevs = set(cl.ancestors(*[cl.rev(n) for n in bases]))

        # Make it easy to refer to self.manifest
        mnfst = self.manifest
        # We don't know which manifests are missing yet
        msng_mnfst_set = {}
        # Nor do we know which filenodes are missing.
        msng_filenode_set = {}

        # A changeset always belongs to itself, so the changenode lookup
        # function for a changenode is identity.
        def identity(x):
            return x

            # so we can include those in the changegroup too.
            #
            # It also remembers which changenode each filenode belongs to.  It
            # does this by assuming the a filenode belongs to the changenode
            # the first manifest that references it belongs to.
            def collect_msng_filenodes(mnfstnode):
                r = mnfst.rev(mnfstnode)
                if mnfst.deltaparent(r) in mnfst.parentrevs(r):
                    # If the previous rev is one of the parents,
                    # we only need to see a diff.
                    deltamf = mnfst.readdelta(mnfstnode)
                    # For each line in the delta
                    for f, fnode in deltamf.iteritems():
                        # And if the file is in the list of files we care
                        # about.
                        if f in changedfiles:
                            # Get the changenode this manifest belongs to
                            clnode = msng_mnfst_set[mnfstnode]
                            # Create the set of filenodes for the file if
                            # there isn't one already.
                            ndset = msng_filenode_set.setdefault(f, {})
                            # And set the filenode's changelog node to the
                            # manifest's if it hasn't been set already.
                            ndset.setdefault(fnode, clnode)
                else:
                    # Otherwise we need a full manifest.
                    m = mnfst.read(mnfstnode)
                    # For every file in we care about.
                    for f in changedfiles:
                        fnode = m.get(f, None)
                        # If it's in the manifest
                        if fnode is not None:
                            # See comments above.
                            clnode = msng_mnfst_set[mnfstnode]
                            ndset = msng_filenode_set.setdefault(f, {})
                            ndset.setdefault(fnode, clnode)
            return collect_msng_filenodes

        # If we determine that a particular file or manifest node must be a
        # node that the recipient of the changegroup will already have, we can
        # also assume the recipient will have all the parents.  This function
        # prunes them from the set of missing nodes.

        # Now that we have all theses utility functions to help out and
        # logically divide up the task, generate the group.
        def gengroup():
            # The set of changed files starts empty.
            changedfiles = set()
            collect = changegroup.collector(cl, msng_mnfst_set, changedfiles)

            # Create a changenode group generator that will call our functions
            # back to lookup the owning changenode and collect information.
            group = cl.group(msng_cl_lst, identity, collect)
            for cnt, chnk in enumerate(group):
                yield chnk
                # revlog.group yields three entries per node, so
                # dividing by 3 gives an approximation of how many
                # nodes have been processed.
                self.ui.progress(_('bundling'), cnt / 3,
                                 unit=_('changesets'))
            changecount = cnt / 3
            self.ui.progress(_('bundling'), None)

            prune(mnfst, msng_mnfst_set)
            msng_mnfst_lst = msng_mnfst_set.keys()
            # Sort the manifestnodes by revision number.
            msng_mnfst_lst.sort(key=mnfst.rev)
            # Create a generator for the manifestnodes that calls our lookup
            # and data collection functions back.
            group = mnfst.group(msng_mnfst_lst,
                                lambda mnode: msng_mnfst_set[mnode],
                                filenode_collector(changedfiles))
            efiles = {}
            for cnt, chnk in enumerate(group):
                if cnt % 3 == 1:
                    mnode = chnk[:20]
                    efiles.update(mnfst.readdelta(mnode))
                yield chnk
                # see above comment for why we divide by 3
                self.ui.progress(_('bundling'), cnt / 3,
                                 unit=_('manifests'), total=changecount)
            self.ui.progress(_('bundling'), None)
            efiles = len(efiles)

            # These are no longer needed, dereference and toss the memory for
            # them.
            msng_mnfst_lst = None
            msng_mnfst_set.clear()

            # Go through all our files in order sorted by name.
            for idx, fname in enumerate(sorted(changedfiles)):
                filerevlog = self.file(fname)
                if not len(filerevlog):
                    raise util.Abort(_("empty or missing revlog for %s") % fname)
                # Toss out the filenodes that the recipient isn't really
                # missing.
                missingfnodes = msng_filenode_set.pop(fname, {})
                prune(filerevlog, missingfnodes)
                # If any filenodes are left, generate the group for them,
                # otherwise don't bother.
                if missingfnodes:
                    yield changegroup.chunkheader(len(fname))

                        yield chnk
            # Signal that no more groups are left.
            yield changegroup.closechunk()
            self.ui.progress(_('bundling'), None)

            if msng_cl_lst:
                self.hook('outgoing', node=hex(msng_cl_lst[0]), source=source)

        return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN')

    def changegroup(self, basenodes, source):
        # to avoid a race we use changegroupsubset() (issue1320)

13706:7beb9834d185

        Another wrinkle is doing the reverse, figuring out which changeset in
        the changegroup a particular filenode or manifestnode belongs to.
        """

        cl = self.changelog
        mf = self.manifest
        mfs = {} # needed manifests
        fnodes = {} # needed file nodes

        # Compute the list of changesets in this changegroup.
        # Some bases may turn out to be superfluous, and some heads may be
        # too.  nodesbetween will return the minimal set of bases and heads
        # necessary to re-create the changegroup.
        if not bases:
            bases = [nullid]
        csets, bases, heads = cl.nodesbetween(bases, heads)

        # can we go through the fast path ?
        heads.sort()
        allheads = self.heads()
        allheads.sort()
        if heads == allheads:
            return self._changegroup(csets, source)

        # slow path
        self.hook('preoutgoing', throw=True, source=source)
        self.changegroupinfo(csets, source)

        # We assume that all ancestors of bases are known
        commonrevs = set(cl.ancestors(*[cl.rev(n) for n in bases]))

        # A changeset always belongs to itself, so the changenode lookup
        # function for a changenode is identity.
        def identity(x):
            return x

            # so we can include those in the changegroup too.
            #
            # It also remembers which changenode each filenode belongs to.  It
            # does this by assuming the a filenode belongs to the changenode
            # the first manifest that references it belongs to.
            def collect(mannode):
                r = mf.rev(mannode)
                if mf.deltaparent(r) in mf.parentrevs(r):
                    # If the previous rev is one of the parents,
                    # we only need to see a diff.
                    deltamf = mf.readdelta(mannode)
                    # For each line in the delta
                    for f, fnode in deltamf.iteritems():
                        # And if the file is in the list of files we care
                        # about.
                        if f in changedfiles:
                            # Get the changenode this manifest belongs to
                            clnode = mfs[mannode]
                            # Create the set of filenodes for the file if
                            # there isn't one already.
                            ndset = fnodes.setdefault(f, {})
                            # And set the filenode's changelog node to the
                            # manifest's if it hasn't been set already.
                            ndset.setdefault(fnode, clnode)
                else:
                    # Otherwise we need a full manifest.
                    m = mf.read(mannode)
                    # For every file in we care about.
                    for f in changedfiles:
                        fnode = m.get(f, None)
                        # If it's in the manifest
                        if fnode is not None:
                            # See comments above.
                            clnode = mfs[mannode]
                            ndset = fnodes.setdefault(f, {})
                            ndset.setdefault(fnode, clnode)
            return collect

        # If we determine that a particular file or manifest node must be a
        # node that the recipient of the changegroup will already have, we can
        # also assume the recipient will have all the parents.  This function
        # prunes them from the set of missing nodes.

        # Now that we have all theses utility functions to help out and
        # logically divide up the task, generate the group.
        def gengroup():
            # The set of changed files starts empty.
            changedfiles = set()
            collect = changegroup.collector(cl, mfs, changedfiles)

            # Create a changenode group generator that will call our functions
            # back to lookup the owning changenode and collect information.
            group = cl.group(csets, identity, collect)
            for cnt, chnk in enumerate(group):
                yield chnk
                # revlog.group yields three entries per node, so
                # dividing by 3 gives an approximation of how many
                # nodes have been processed.
                self.ui.progress(_('bundling'), cnt / 3,
                                 unit=_('changesets'))
            changecount = cnt / 3
            self.ui.progress(_('bundling'), None)

            prune(mf, mfs)
            # Create a generator for the manifestnodes that calls our lookup
            # and data collection functions back.
            group = mf.group(sorted(mfs, key=mf.rev),
                             lambda mnode: mfs[mnode],
                             filenode_collector(changedfiles))
            efiles = {}
            for cnt, chnk in enumerate(group):
                if cnt % 3 == 1:
                    mnode = chnk[:20]
                    efiles.update(mf.readdelta(mnode))
                yield chnk
                # see above comment for why we divide by 3
                self.ui.progress(_('bundling'), cnt / 3,
                                 unit=_('manifests'), total=changecount)
            self.ui.progress(_('bundling'), None)
            efiles = len(efiles)

            mfs.clear()

            # Go through all our files in order sorted by name.
            for idx, fname in enumerate(sorted(changedfiles)):
                filerevlog = self.file(fname)
                if not len(filerevlog):
                    raise util.Abort(_("empty or missing revlog for %s") % fname)
                # Toss out the filenodes that the recipient isn't really
                # missing.
                missingfnodes = fnodes.pop(fname, {})
                prune(filerevlog, missingfnodes)
                # If any filenodes are left, generate the group for them,
                # otherwise don't bother.
                if missingfnodes:
                    yield changegroup.chunkheader(len(fname))

                        yield chnk
            # Signal that no more groups are left.
            yield changegroup.closechunk()
            self.ui.progress(_('bundling'), None)

            if csets:
                self.hook('outgoing', node=hex(csets[0]), source=source)

        return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN')

    def changegroup(self, basenodes, source):
        # to avoid a race we use changegroupsubset() (issue1320)