Mercurial > hg
comparison mercurial/changegroup.py @ 40344:2c5835b4246b
narrow: when widening, don't include manifests the client already has
When widening, we already don't include the changelog (since
f1844a10ee19) and files that the client already has (since
c73c7653dfb9). However, we still include all manifests needed for the
new narrowspec. When using flat manifests, that means we resend all
the manifests even though the client necessarily has all of them. For
tree manifests, we unnecessarily resend the root manifests and any
subdirectory manifests that the client already has.
This patch makes it so we no longer resend manifests that the client
already has. It does so by passing an extra matcher to the changegroup
packer and it uses that for filtering out directories matching the old
matcher's visitdir(). For consistency between directories and files,
it also makes the filtering of files look at both old and new matcher
rather than passing in a diff matcher as we did before.
Differential Revision: https://phab.mercurial-scm.org/D4895
author | Martin von Zweigbergk <martinvonz@google.com> |
---|---|
date | Fri, 05 Oct 2018 11:07:34 -0700 |
parents | 4fd0fac48922 |
children | 6a917075535a |
comparison
equal
deleted
inserted
replaced
40343:a69d5823af6d | 40344:2c5835b4246b |
---|---|
725 | 725 |
726 if progress: | 726 if progress: |
727 progress.complete() | 727 progress.complete() |
728 | 728 |
729 class cgpacker(object): | 729 class cgpacker(object): |
730 def __init__(self, repo, filematcher, version, | 730 def __init__(self, repo, oldmatcher, matcher, version, |
731 builddeltaheader, manifestsend, | 731 builddeltaheader, manifestsend, |
732 forcedeltaparentprev=False, | 732 forcedeltaparentprev=False, |
733 bundlecaps=None, ellipses=False, | 733 bundlecaps=None, ellipses=False, |
734 shallow=False, ellipsisroots=None, fullnodes=None): | 734 shallow=False, ellipsisroots=None, fullnodes=None): |
735 """Given a source repo, construct a bundler. | 735 """Given a source repo, construct a bundler. |
736 | 736 |
737 filematcher is a matcher that matches on files to include in the | 737 oldmatcher is a matcher that matches on files the client already has. |
738 These will not be included in the changegroup. | |
739 | |
740 matcher is a matcher that matches on files to include in the | |
738 changegroup. Used to facilitate sparse changegroups. | 741 changegroup. Used to facilitate sparse changegroups. |
739 | 742 |
740 forcedeltaparentprev indicates whether delta parents must be against | 743 forcedeltaparentprev indicates whether delta parents must be against |
741 the previous revision in a delta group. This should only be used for | 744 the previous revision in a delta group. This should only be used for |
742 compatibility with changegroup version 1. | 745 compatibility with changegroup version 1. |
759 fullnodes is the set of changelog nodes which should not be ellipsis | 762 fullnodes is the set of changelog nodes which should not be ellipsis |
760 nodes. We store this rather than the set of nodes that should be | 763 nodes. We store this rather than the set of nodes that should be |
761 ellipsis because for very large histories we expect this to be | 764 ellipsis because for very large histories we expect this to be |
762 significantly smaller. | 765 significantly smaller. |
763 """ | 766 """ |
764 assert filematcher | 767 assert oldmatcher |
765 self._filematcher = filematcher | 768 assert matcher |
769 self._oldmatcher = oldmatcher | |
770 self._matcher = matcher | |
766 | 771 |
767 self.version = version | 772 self.version = version |
768 self._forcedeltaparentprev = forcedeltaparentprev | 773 self._forcedeltaparentprev = forcedeltaparentprev |
769 self._builddeltaheader = builddeltaheader | 774 self._builddeltaheader = builddeltaheader |
770 self._manifestsend = manifestsend | 775 self._manifestsend = manifestsend |
1025 | 1030 |
1026 while tmfnodes: | 1031 while tmfnodes: |
1027 tree, nodes = tmfnodes.popitem() | 1032 tree, nodes = tmfnodes.popitem() |
1028 store = mfl.getstorage(tree) | 1033 store = mfl.getstorage(tree) |
1029 | 1034 |
1030 if not self._filematcher.visitdir(store.tree[:-1] or '.'): | 1035 if not self._matcher.visitdir(store.tree[:-1] or '.'): |
1031 # No nodes to send because this directory is out of | 1036 # No nodes to send because this directory is out of |
1032 # the client's view of the repository (probably | 1037 # the client's view of the repository (probably |
1033 # because of narrow clones). | 1038 # because of narrow clones). |
1034 prunednodes = [] | 1039 prunednodes = [] |
1035 else: | 1040 else: |
1049 topic=_('manifests'), | 1054 topic=_('manifests'), |
1050 clrevtolocalrev=clrevtolocalrev, | 1055 clrevtolocalrev=clrevtolocalrev, |
1051 fullclnodes=self._fullclnodes, | 1056 fullclnodes=self._fullclnodes, |
1052 precomputedellipsis=self._precomputedellipsis) | 1057 precomputedellipsis=self._precomputedellipsis) |
1053 | 1058 |
1054 yield tree, deltas | 1059 if not self._oldmatcher.visitdir(store.tree[:-1] or '.'): |
1060 yield tree, deltas | |
1061 else: | |
1062 # 'deltas' is a generator and we need to consume it even if | |
1063 # we are not going to send it because a side-effect is that | |
1064 # it updates tmdnodes (via lookupfn) | |
1065 for d in deltas: | |
1066 pass | |
1067 if not tree: | |
1068 yield tree, [] | |
1055 | 1069 |
1056 def _prunemanifests(self, store, nodes, commonrevs): | 1070 def _prunemanifests(self, store, nodes, commonrevs): |
1057 # This is split out as a separate method to allow filtering | 1071 # This is split out as a separate method to allow filtering |
1058 # commonrevs in extension code. | 1072 # commonrevs in extension code. |
1059 # | 1073 # |
1064 return [n for n in nodes if flr(frev(n)) not in commonrevs] | 1078 return [n for n in nodes if flr(frev(n)) not in commonrevs] |
1065 | 1079 |
1066 # The 'source' parameter is useful for extensions | 1080 # The 'source' parameter is useful for extensions |
1067 def generatefiles(self, changedfiles, commonrevs, source, | 1081 def generatefiles(self, changedfiles, commonrevs, source, |
1068 mfdicts, fastpathlinkrev, fnodes, clrevs): | 1082 mfdicts, fastpathlinkrev, fnodes, clrevs): |
1069 changedfiles = list(filter(self._filematcher, changedfiles)) | 1083 changedfiles = [f for f in changedfiles |
1084 if self._matcher(f) and not self._oldmatcher(f)] | |
1070 | 1085 |
1071 if not fastpathlinkrev: | 1086 if not fastpathlinkrev: |
1072 def normallinknodes(unused, fname): | 1087 def normallinknodes(unused, fname): |
1073 return fnodes.get(fname, {}) | 1088 return fnodes.get(fname, {}) |
1074 else: | 1089 else: |
1149 | 1164 |
1150 yield fname, deltas | 1165 yield fname, deltas |
1151 | 1166 |
1152 progress.complete() | 1167 progress.complete() |
1153 | 1168 |
1154 def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False, | 1169 def _makecg1packer(repo, oldmatcher, matcher, bundlecaps, |
1155 shallow=False, ellipsisroots=None, fullnodes=None): | 1170 ellipses=False, shallow=False, ellipsisroots=None, |
1171 fullnodes=None): | |
1156 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( | 1172 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( |
1157 d.node, d.p1node, d.p2node, d.linknode) | 1173 d.node, d.p1node, d.p2node, d.linknode) |
1158 | 1174 |
1159 return cgpacker(repo, filematcher, b'01', | 1175 return cgpacker(repo, oldmatcher, matcher, b'01', |
1160 builddeltaheader=builddeltaheader, | 1176 builddeltaheader=builddeltaheader, |
1161 manifestsend=b'', | 1177 manifestsend=b'', |
1162 forcedeltaparentprev=True, | 1178 forcedeltaparentprev=True, |
1163 bundlecaps=bundlecaps, | 1179 bundlecaps=bundlecaps, |
1164 ellipses=ellipses, | 1180 ellipses=ellipses, |
1165 shallow=shallow, | 1181 shallow=shallow, |
1166 ellipsisroots=ellipsisroots, | 1182 ellipsisroots=ellipsisroots, |
1167 fullnodes=fullnodes) | 1183 fullnodes=fullnodes) |
1168 | 1184 |
1169 def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False, | 1185 def _makecg2packer(repo, oldmatcher, matcher, bundlecaps, |
1170 shallow=False, ellipsisroots=None, fullnodes=None): | 1186 ellipses=False, shallow=False, ellipsisroots=None, |
1187 fullnodes=None): | |
1171 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( | 1188 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( |
1172 d.node, d.p1node, d.p2node, d.basenode, d.linknode) | 1189 d.node, d.p1node, d.p2node, d.basenode, d.linknode) |
1173 | 1190 |
1174 return cgpacker(repo, filematcher, b'02', | 1191 return cgpacker(repo, oldmatcher, matcher, b'02', |
1175 builddeltaheader=builddeltaheader, | 1192 builddeltaheader=builddeltaheader, |
1176 manifestsend=b'', | 1193 manifestsend=b'', |
1177 bundlecaps=bundlecaps, | 1194 bundlecaps=bundlecaps, |
1178 ellipses=ellipses, | 1195 ellipses=ellipses, |
1179 shallow=shallow, | 1196 shallow=shallow, |
1180 ellipsisroots=ellipsisroots, | 1197 ellipsisroots=ellipsisroots, |
1181 fullnodes=fullnodes) | 1198 fullnodes=fullnodes) |
1182 | 1199 |
1183 def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False, | 1200 def _makecg3packer(repo, oldmatcher, matcher, bundlecaps, |
1184 shallow=False, ellipsisroots=None, fullnodes=None): | 1201 ellipses=False, shallow=False, ellipsisroots=None, |
1202 fullnodes=None): | |
1185 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( | 1203 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( |
1186 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) | 1204 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) |
1187 | 1205 |
1188 return cgpacker(repo, filematcher, b'03', | 1206 return cgpacker(repo, oldmatcher, matcher, b'03', |
1189 builddeltaheader=builddeltaheader, | 1207 builddeltaheader=builddeltaheader, |
1190 manifestsend=closechunk(), | 1208 manifestsend=closechunk(), |
1191 bundlecaps=bundlecaps, | 1209 bundlecaps=bundlecaps, |
1192 ellipses=ellipses, | 1210 ellipses=ellipses, |
1193 shallow=shallow, | 1211 shallow=shallow, |
1250 if 'generaldelta' in repo.requirements: | 1268 if 'generaldelta' in repo.requirements: |
1251 versions.discard('01') | 1269 versions.discard('01') |
1252 assert versions | 1270 assert versions |
1253 return min(versions) | 1271 return min(versions) |
1254 | 1272 |
1255 def getbundler(version, repo, bundlecaps=None, filematcher=None, | 1273 def getbundler(version, repo, bundlecaps=None, oldmatcher=None, |
1256 ellipses=False, shallow=False, ellipsisroots=None, | 1274 matcher=None, ellipses=False, shallow=False, |
1257 fullnodes=None): | 1275 ellipsisroots=None, fullnodes=None): |
1258 assert version in supportedoutgoingversions(repo) | 1276 assert version in supportedoutgoingversions(repo) |
1259 | 1277 |
1260 if filematcher is None: | 1278 if matcher is None: |
1261 filematcher = matchmod.alwaysmatcher(repo.root, '') | 1279 matcher = matchmod.alwaysmatcher(repo.root, '') |
1262 | 1280 if oldmatcher is None: |
1263 if version == '01' and not filematcher.always(): | 1281 oldmatcher = matchmod.nevermatcher(repo.root, '') |
1282 | |
1283 if version == '01' and not matcher.always(): | |
1264 raise error.ProgrammingError('version 01 changegroups do not support ' | 1284 raise error.ProgrammingError('version 01 changegroups do not support ' |
1265 'sparse file matchers') | 1285 'sparse file matchers') |
1266 | 1286 |
1267 if ellipses and version in (b'01', b'02'): | 1287 if ellipses and version in (b'01', b'02'): |
1268 raise error.Abort( | 1288 raise error.Abort( |
1269 _('ellipsis nodes require at least cg3 on client and server, ' | 1289 _('ellipsis nodes require at least cg3 on client and server, ' |
1270 'but negotiated version %s') % version) | 1290 'but negotiated version %s') % version) |
1271 | 1291 |
1272 # Requested files could include files not in the local store. So | 1292 # Requested files could include files not in the local store. So |
1273 # filter those out. | 1293 # filter those out. |
1274 filematcher = repo.narrowmatch(filematcher) | 1294 matcher = repo.narrowmatch(matcher) |
1275 | 1295 |
1276 fn = _packermap[version][0] | 1296 fn = _packermap[version][0] |
1277 return fn(repo, filematcher, bundlecaps, ellipses=ellipses, | 1297 return fn(repo, oldmatcher, matcher, bundlecaps, ellipses=ellipses, |
1278 shallow=shallow, ellipsisroots=ellipsisroots, | 1298 shallow=shallow, ellipsisroots=ellipsisroots, |
1279 fullnodes=fullnodes) | 1299 fullnodes=fullnodes) |
1280 | 1300 |
1281 def getunbundler(version, fh, alg, extras=None): | 1301 def getunbundler(version, fh, alg, extras=None): |
1282 return _packermap[version][1](fh, alg, extras=extras) | 1302 return _packermap[version][1](fh, alg, extras=extras) |
1295 fastpath=fastpath, bundlecaps=bundlecaps) | 1315 fastpath=fastpath, bundlecaps=bundlecaps) |
1296 return getunbundler(version, util.chunkbuffer(cgstream), None, | 1316 return getunbundler(version, util.chunkbuffer(cgstream), None, |
1297 {'clcount': len(outgoing.missing) }) | 1317 {'clcount': len(outgoing.missing) }) |
1298 | 1318 |
1299 def makestream(repo, outgoing, version, source, fastpath=False, | 1319 def makestream(repo, outgoing, version, source, fastpath=False, |
1300 bundlecaps=None, filematcher=None): | 1320 bundlecaps=None, matcher=None): |
1301 bundler = getbundler(version, repo, bundlecaps=bundlecaps, | 1321 bundler = getbundler(version, repo, bundlecaps=bundlecaps, |
1302 filematcher=filematcher) | 1322 matcher=matcher) |
1303 | 1323 |
1304 repo = repo.unfiltered() | 1324 repo = repo.unfiltered() |
1305 commonrevs = outgoing.common | 1325 commonrevs = outgoing.common |
1306 csets = outgoing.missing | 1326 csets = outgoing.missing |
1307 heads = outgoing.missingheads | 1327 heads = outgoing.missingheads |