Mercurial > hg
changeset 40344:2c5835b4246b
narrow: when widening, don't include manifests the client already has
When widening, we already don't include the changelog (since
f1844a10ee19) and files that the client already has (since
c73c7653dfb9). However, we still include all manifests needed for the
new narrowspec. When using flat manifests, that means we resend all
the manifests even though the client necessarily has all of them. For
tree manifests, we unnecessarily resend the root manifests and any
subdirectory manifests that the client already has.
This patch makes it so we no longer resend manifests that the client
already has. It does so by passing an extra matcher to the changegroup
packer and it uses that for filtering out directories matching the old
matcher's visitdir(). For consistency between directories and files,
it also makes the filtering of files look at both old and new matcher
rather than passing in a diff matcher as we did before.
Differential Revision: https://phab.mercurial-scm.org/D4895
| author | Martin von Zweigbergk <martinvonz@google.com> |
|---|---|
| date | Fri, 05 Oct 2018 11:07:34 -0700 |
| parents | a69d5823af6d |
| children | d30a19d10441 |
| files | hgext/narrow/narrowbundle2.py hgext/narrow/narrowwirepeer.py mercurial/bundle2.py mercurial/changegroup.py mercurial/exchange.py |
| diffstat | 5 files changed, 58 insertions(+), 40 deletions(-) [+] |
line wrap: on
line diff
--- a/hgext/narrow/narrowbundle2.py Wed Oct 17 09:30:07 2018 -0700 +++ b/hgext/narrow/narrowbundle2.py Fri Oct 05 11:07:34 2018 -0700 @@ -117,7 +117,7 @@ repo, set(), common, known, newmatch) if newvisit: packer = changegroup.getbundler(version, repo, - filematcher=newmatch, + matcher=newmatch, ellipses=True, shallow=depth is not None, ellipsisroots=newellipsis, @@ -135,7 +135,7 @@ repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes)) if visitnodes: packer = changegroup.getbundler(version, repo, - filematcher=newmatch, + matcher=newmatch, ellipses=True, shallow=depth is not None, ellipsisroots=ellipsisroots,
--- a/hgext/narrow/narrowwirepeer.py Wed Oct 17 09:30:07 2018 -0700 +++ b/hgext/narrow/narrowwirepeer.py Fri Oct 05 11:07:34 2018 -0700 @@ -12,7 +12,6 @@ error, extensions, hg, - match as matchmod, narrowspec, pycompat, wireprototypes, @@ -82,9 +81,8 @@ exclude=newexcludes) oldmatch = narrowspec.match(repo.root, include=oldincludes, exclude=oldexcludes) - diffmatch = matchmod.differencematcher(newmatch, oldmatch) - bundler = bundle2.widen_bundle(repo, diffmatch, common, known, + bundler = bundle2.widen_bundle(repo, oldmatch, newmatch, common, known, cgversion, ellipses) except error.Abort as exc: bundler = bundle2.bundle20(repo.ui)
--- a/mercurial/bundle2.py Wed Oct 17 09:30:07 2018 -0700 +++ b/mercurial/bundle2.py Fri Oct 05 11:07:34 2018 -0700 @@ -2278,12 +2278,13 @@ streamclone.applybundlev2(repo, part, filecount, bytecount, requirements) -def widen_bundle(repo, diffmatcher, common, known, cgversion, ellipses): +def widen_bundle(repo, oldmatcher, newmatcher, common, known, cgversion, + ellipses): """generates bundle2 for widening a narrow clone repo is the localrepository instance - diffmatcher is a differencemacther of '(newincludes, newexcludes) - - (oldincludes, oldexcludes)' + oldmatcher matches what the client already has + newmatcher matches what the client needs (including what it already has) common is set of common heads between server and client known is a set of revs known on the client side (used in ellipses) cgversion is the changegroup version to send @@ -2300,7 +2301,8 @@ # XXX: we should only send the filelogs (and treemanifest). user # already has the changelog and manifest packer = changegroup.getbundler(cgversion, repo, - filematcher=diffmatcher, + oldmatcher=oldmatcher, + matcher=newmatcher, fullnodes=commonnodes) cgdata = packer.generate(set([nodemod.nullid]), list(commonnodes), False, 'narrow_widen', changelog=False)
--- a/mercurial/changegroup.py Wed Oct 17 09:30:07 2018 -0700 +++ b/mercurial/changegroup.py Fri Oct 05 11:07:34 2018 -0700 @@ -727,14 +727,17 @@ progress.complete() class cgpacker(object): - def __init__(self, repo, filematcher, version, + def __init__(self, repo, oldmatcher, matcher, version, builddeltaheader, manifestsend, forcedeltaparentprev=False, bundlecaps=None, ellipses=False, shallow=False, ellipsisroots=None, fullnodes=None): """Given a source repo, construct a bundler. - filematcher is a matcher that matches on files to include in the + oldmatcher is a matcher that matches on files the client already has. + These will not be included in the changegroup. + + matcher is a matcher that matches on files to include in the changegroup. Used to facilitate sparse changegroups. forcedeltaparentprev indicates whether delta parents must be against @@ -761,8 +764,10 @@ ellipsis because for very large histories we expect this to be significantly smaller. """ - assert filematcher - self._filematcher = filematcher + assert oldmatcher + assert matcher + self._oldmatcher = oldmatcher + self._matcher = matcher self.version = version self._forcedeltaparentprev = forcedeltaparentprev @@ -1027,7 +1032,7 @@ tree, nodes = tmfnodes.popitem() store = mfl.getstorage(tree) - if not self._filematcher.visitdir(store.tree[:-1] or '.'): + if not self._matcher.visitdir(store.tree[:-1] or '.'): # No nodes to send because this directory is out of # the client's view of the repository (probably # because of narrow clones). @@ -1051,7 +1056,16 @@ fullclnodes=self._fullclnodes, precomputedellipsis=self._precomputedellipsis) - yield tree, deltas + if not self._oldmatcher.visitdir(store.tree[:-1] or '.'): + yield tree, deltas + else: + # 'deltas' is a generator and we need to consume it even if + # we are not going to send it because a side-effect is that + # it updates tmdnodes (via lookupfn) + for d in deltas: + pass + if not tree: + yield tree, [] def _prunemanifests(self, store, nodes, commonrevs): # This is split out as a separate method to allow filtering @@ -1066,7 +1080,8 @@ # The 'source' parameter is useful for extensions def generatefiles(self, changedfiles, commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs): - changedfiles = list(filter(self._filematcher, changedfiles)) + changedfiles = [f for f in changedfiles + if self._matcher(f) and not self._oldmatcher(f)] if not fastpathlinkrev: def normallinknodes(unused, fname): @@ -1151,12 +1166,13 @@ progress.complete() -def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False, - shallow=False, ellipsisroots=None, fullnodes=None): +def _makecg1packer(repo, oldmatcher, matcher, bundlecaps, + ellipses=False, shallow=False, ellipsisroots=None, + fullnodes=None): builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.linknode) - return cgpacker(repo, filematcher, b'01', + return cgpacker(repo, oldmatcher, matcher, b'01', builddeltaheader=builddeltaheader, manifestsend=b'', forcedeltaparentprev=True, @@ -1166,12 +1182,13 @@ ellipsisroots=ellipsisroots, fullnodes=fullnodes) -def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False, - shallow=False, ellipsisroots=None, fullnodes=None): +def _makecg2packer(repo, oldmatcher, matcher, bundlecaps, + ellipses=False, shallow=False, ellipsisroots=None, + fullnodes=None): builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.basenode, d.linknode) - return cgpacker(repo, filematcher, b'02', + return cgpacker(repo, oldmatcher, matcher, b'02', builddeltaheader=builddeltaheader, manifestsend=b'', bundlecaps=bundlecaps, @@ -1180,12 +1197,13 @@ ellipsisroots=ellipsisroots, fullnodes=fullnodes) -def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False, - shallow=False, ellipsisroots=None, fullnodes=None): +def _makecg3packer(repo, oldmatcher, matcher, bundlecaps, + ellipses=False, shallow=False, ellipsisroots=None, + fullnodes=None): builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) - return cgpacker(repo, filematcher, b'03', + return cgpacker(repo, oldmatcher, matcher, b'03', builddeltaheader=builddeltaheader, manifestsend=closechunk(), bundlecaps=bundlecaps, @@ -1252,15 +1270,17 @@ assert versions return min(versions) -def getbundler(version, repo, bundlecaps=None, filematcher=None, - ellipses=False, shallow=False, ellipsisroots=None, - fullnodes=None): +def getbundler(version, repo, bundlecaps=None, oldmatcher=None, + matcher=None, ellipses=False, shallow=False, + ellipsisroots=None, fullnodes=None): assert version in supportedoutgoingversions(repo) - if filematcher is None: - filematcher = matchmod.alwaysmatcher(repo.root, '') + if matcher is None: + matcher = matchmod.alwaysmatcher(repo.root, '') + if oldmatcher is None: + oldmatcher = matchmod.nevermatcher(repo.root, '') - if version == '01' and not filematcher.always(): + if version == '01' and not matcher.always(): raise error.ProgrammingError('version 01 changegroups do not support ' 'sparse file matchers') @@ -1271,10 +1291,10 @@ # Requested files could include files not in the local store. So # filter those out. - filematcher = repo.narrowmatch(filematcher) + matcher = repo.narrowmatch(matcher) fn = _packermap[version][0] - return fn(repo, filematcher, bundlecaps, ellipses=ellipses, + return fn(repo, oldmatcher, matcher, bundlecaps, ellipses=ellipses, shallow=shallow, ellipsisroots=ellipsisroots, fullnodes=fullnodes) @@ -1297,9 +1317,9 @@ {'clcount': len(outgoing.missing) }) def makestream(repo, outgoing, version, source, fastpath=False, - bundlecaps=None, filematcher=None): + bundlecaps=None, matcher=None): bundler = getbundler(version, repo, bundlecaps=bundlecaps, - filematcher=filematcher) + matcher=matcher) repo = repo.unfiltered() commonrevs = outgoing.common
--- a/mercurial/exchange.py Wed Oct 17 09:30:07 2018 -0700 +++ b/mercurial/exchange.py Fri Oct 05 11:07:34 2018 -0700 @@ -2153,14 +2153,12 @@ if kwargs.get(r'narrow', False): include = sorted(filter(bool, kwargs.get(r'includepats', []))) exclude = sorted(filter(bool, kwargs.get(r'excludepats', []))) - filematcher = narrowspec.match(repo.root, include=include, - exclude=exclude) + matcher = narrowspec.match(repo.root, include=include, exclude=exclude) else: - filematcher = None + matcher = None cgstream = changegroup.makestream(repo, outgoing, version, source, - bundlecaps=bundlecaps, - filematcher=filematcher) + bundlecaps=bundlecaps, matcher=matcher) part = bundler.newpart('changegroup', data=cgstream) if cgversions: