Mercurial > hg
comparison mercurial/changegroup.py @ 38908:1469584ad5fe
changegroup: specify ellipses mode explicitly
Currently, code throughout changegroup relies on the presence
of self._full_nodes to enable ellipses mode. This is a very tenuous
check. And the check may be wrong once we move _full_nodes into
cgpacker.
Let's capture the enabling of ellipses mode explicitly as a constructor
argument and as an instance variable.
We could probably derive ellipses mode by presence of other
variables. But for now, this explicit approach seems simplest
since it is most compatible with existing code.
Differential Revision: https://phab.mercurial-scm.org/D4090
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Fri, 03 Aug 2018 14:00:18 -0700 |
parents | ad4c4cc9a5ac |
children | 1af339c22aeb |
comparison
equal
deleted
inserted
replaced
38907:ad4c4cc9a5ac | 38908:1469584ad5fe |
---|---|
520 deltachunks = attr.ib() | 520 deltachunks = attr.ib() |
521 | 521 |
522 class cgpacker(object): | 522 class cgpacker(object): |
523 def __init__(self, repo, filematcher, version, allowreorder, | 523 def __init__(self, repo, filematcher, version, allowreorder, |
524 useprevdelta, builddeltaheader, manifestsend, | 524 useprevdelta, builddeltaheader, manifestsend, |
525 sendtreemanifests, bundlecaps=None, shallow=False, | 525 sendtreemanifests, bundlecaps=None, ellipses=False, |
526 ellipsisroots=None): | 526 shallow=False, ellipsisroots=None): |
527 """Given a source repo, construct a bundler. | 527 """Given a source repo, construct a bundler. |
528 | 528 |
529 filematcher is a matcher that matches on files to include in the | 529 filematcher is a matcher that matches on files to include in the |
530 changegroup. Used to facilitate sparse changegroups. | 530 changegroup. Used to facilitate sparse changegroups. |
531 | 531 |
540 delta. | 540 delta. |
541 | 541 |
542 manifestsend is a chunk to send after manifests have been fully emitted. | 542 manifestsend is a chunk to send after manifests have been fully emitted. |
543 | 543 |
544 sendtreemanifests indicates whether tree manifests should be emitted. | 544 sendtreemanifests indicates whether tree manifests should be emitted. |
545 | |
546 ellipses indicates whether ellipsis serving mode is enabled. | |
545 | 547 |
546 bundlecaps is optional and can be used to specify the set of | 548 bundlecaps is optional and can be used to specify the set of |
547 capabilities which can be used to build the bundle. While bundlecaps is | 549 capabilities which can be used to build the bundle. While bundlecaps is |
548 unused in core Mercurial, extensions rely on this feature to communicate | 550 unused in core Mercurial, extensions rely on this feature to communicate |
549 capabilities to customize the changegroup packer. | 551 capabilities to customize the changegroup packer. |
557 self.version = version | 559 self.version = version |
558 self._useprevdelta = useprevdelta | 560 self._useprevdelta = useprevdelta |
559 self._builddeltaheader = builddeltaheader | 561 self._builddeltaheader = builddeltaheader |
560 self._manifestsend = manifestsend | 562 self._manifestsend = manifestsend |
561 self._sendtreemanifests = sendtreemanifests | 563 self._sendtreemanifests = sendtreemanifests |
564 self._ellipses = ellipses | |
562 | 565 |
563 # Set of capabilities we can use to build the bundle. | 566 # Set of capabilities we can use to build the bundle. |
564 if bundlecaps is None: | 567 if bundlecaps is None: |
565 bundlecaps = set() | 568 bundlecaps = set() |
566 self._bundlecaps = bundlecaps | 569 self._bundlecaps = bundlecaps |
626 # The one invariant we *know* holds is that the new (potentially | 629 # The one invariant we *know* holds is that the new (potentially |
627 # bogus) DAG shape will be valid if we order the nodes in the | 630 # bogus) DAG shape will be valid if we order the nodes in the |
628 # order that they're introduced in dramatis personae by the | 631 # order that they're introduced in dramatis personae by the |
629 # changelog, so what we do is we sort the non-changelog histories | 632 # changelog, so what we do is we sort the non-changelog histories |
630 # by the order in which they are used by the changelog. | 633 # by the order in which they are used by the changelog. |
631 if util.safehasattr(self, '_full_nodes') and self._clnodetorev: | 634 if self._ellipses and self._clnodetorev: |
632 key = lambda n: self._clnodetorev[lookup(n)] | 635 key = lambda n: self._clnodetorev[lookup(n)] |
633 return [store.rev(n) for n in sorted(nodelist, key=key)] | 636 return [store.rev(n) for n in sorted(nodelist, key=key)] |
634 | 637 |
635 # for generaldelta revlogs, we linearize the revs; this will both be | 638 # for generaldelta revlogs, we linearize the revs; this will both be |
636 # much quicker and generate a much smaller bundle | 639 # much quicker and generate a much smaller bundle |
727 mfl = repo.manifestlog | 730 mfl = repo.manifestlog |
728 # TODO violates storage abstraction. | 731 # TODO violates storage abstraction. |
729 mfrevlog = mfl._revlog | 732 mfrevlog = mfl._revlog |
730 changedfiles = set() | 733 changedfiles = set() |
731 | 734 |
732 ellipsesmode = util.safehasattr(self, '_full_nodes') | |
733 | |
734 # Callback for the changelog, used to collect changed files and | 735 # Callback for the changelog, used to collect changed files and |
735 # manifest nodes. | 736 # manifest nodes. |
736 # Returns the linkrev node (identity in the changelog case). | 737 # Returns the linkrev node (identity in the changelog case). |
737 def lookupcl(x): | 738 def lookupcl(x): |
738 c = cl.read(x) | 739 c = cl.read(x) |
739 clrevorder[x] = len(clrevorder) | 740 clrevorder[x] = len(clrevorder) |
740 | 741 |
741 if ellipsesmode: | 742 if self._ellipses: |
742 # Only update mfs if x is going to be sent. Otherwise we | 743 # Only update mfs if x is going to be sent. Otherwise we |
743 # end up with bogus linkrevs specified for manifests and | 744 # end up with bogus linkrevs specified for manifests and |
744 # we skip some manifest nodes that we should otherwise | 745 # we skip some manifest nodes that we should otherwise |
745 # have sent. | 746 # have sent. |
746 if (x in self._full_nodes | 747 if (x in self._full_nodes |
803 | 804 |
804 for chunk in self.generatemanifests(commonrevs, clrevorder, | 805 for chunk in self.generatemanifests(commonrevs, clrevorder, |
805 fastpathlinkrev, mfs, fnodes, source): | 806 fastpathlinkrev, mfs, fnodes, source): |
806 yield chunk | 807 yield chunk |
807 | 808 |
808 if ellipsesmode: | 809 if self._ellipses: |
809 mfdicts = None | 810 mfdicts = None |
810 if self._isshallow: | 811 if self._isshallow: |
811 mfdicts = [(self._repo.manifestlog[n].read(), lr) | 812 mfdicts = [(self._repo.manifestlog[n].read(), lr) |
812 for (n, lr) in mfs.iteritems()] | 813 for (n, lr) in mfs.iteritems()] |
813 | 814 |
823 llr = filerevlog.linkrev | 824 llr = filerevlog.linkrev |
824 fln = filerevlog.node | 825 fln = filerevlog.node |
825 revs = ((r, llr(r)) for r in filerevlog) | 826 revs = ((r, llr(r)) for r in filerevlog) |
826 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs) | 827 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs) |
827 | 828 |
828 if ellipsesmode: | 829 if self._ellipses: |
829 # We need to pass the mfdicts variable down into | 830 # We need to pass the mfdicts variable down into |
830 # generatefiles(), but more than one command might have | 831 # generatefiles(), but more than one command might have |
831 # wrapped generatefiles so we can't modify the function | 832 # wrapped generatefiles so we can't modify the function |
832 # signature. Instead, we pass the data to ourselves using an | 833 # signature. Instead, we pass the data to ourselves using an |
833 # instance attribute. I'm sorry. | 834 # instance attribute. I'm sorry. |
980 raise error.ProgrammingError( | 981 raise error.ProgrammingError( |
981 'cg1 should not be used in this case') | 982 'cg1 should not be used in this case') |
982 return prev | 983 return prev |
983 | 984 |
984 # Narrow ellipses mode. | 985 # Narrow ellipses mode. |
985 if util.safehasattr(self, '_full_nodes'): | 986 if self._ellipses: |
986 # TODO: send better deltas when in narrow mode. | 987 # TODO: send better deltas when in narrow mode. |
987 # | 988 # |
988 # changegroup.group() loops over revisions to send, | 989 # changegroup.group() loops over revisions to send, |
989 # including revisions we'll skip. What this means is that | 990 # including revisions we'll skip. What this means is that |
990 # `prev` will be a potentially useless delta base for all | 991 # `prev` will be a potentially useless delta base for all |
1020 base = nullrev | 1021 base = nullrev |
1021 | 1022 |
1022 return base | 1023 return base |
1023 | 1024 |
1024 def _revchunk(self, store, rev, prev, linknode): | 1025 def _revchunk(self, store, rev, prev, linknode): |
1025 if util.safehasattr(self, '_full_nodes'): | 1026 if self._ellipses: |
1026 fn = self._revisiondeltanarrow | 1027 fn = self._revisiondeltanarrow |
1027 else: | 1028 else: |
1028 fn = self._revisiondeltanormal | 1029 fn = self._revisiondeltanormal |
1029 | 1030 |
1030 delta = fn(store, rev, prev, linknode) | 1031 delta = fn(store, rev, prev, linknode) |
1190 linknode=linknode, | 1191 linknode=linknode, |
1191 flags=flags, | 1192 flags=flags, |
1192 deltachunks=(diffheader, data), | 1193 deltachunks=(diffheader, data), |
1193 ) | 1194 ) |
1194 | 1195 |
1195 def _makecg1packer(repo, filematcher, bundlecaps, shallow=False, | 1196 def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False, |
1196 ellipsisroots=None): | 1197 shallow=False, ellipsisroots=None): |
1197 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( | 1198 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( |
1198 d.node, d.p1node, d.p2node, d.linknode) | 1199 d.node, d.p1node, d.p2node, d.linknode) |
1199 | 1200 |
1200 return cgpacker(repo, filematcher, b'01', | 1201 return cgpacker(repo, filematcher, b'01', |
1201 useprevdelta=True, | 1202 useprevdelta=True, |
1202 allowreorder=None, | 1203 allowreorder=None, |
1203 builddeltaheader=builddeltaheader, | 1204 builddeltaheader=builddeltaheader, |
1204 manifestsend=b'', | 1205 manifestsend=b'', |
1205 sendtreemanifests=False, | 1206 sendtreemanifests=False, |
1206 bundlecaps=bundlecaps, | 1207 bundlecaps=bundlecaps, |
1208 ellipses=ellipses, | |
1207 shallow=shallow, | 1209 shallow=shallow, |
1208 ellipsisroots=ellipsisroots) | 1210 ellipsisroots=ellipsisroots) |
1209 | 1211 |
1210 def _makecg2packer(repo, filematcher, bundlecaps, shallow=False, | 1212 def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False, |
1211 ellipsisroots=None): | 1213 shallow=False, ellipsisroots=None): |
1212 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( | 1214 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( |
1213 d.node, d.p1node, d.p2node, d.basenode, d.linknode) | 1215 d.node, d.p1node, d.p2node, d.basenode, d.linknode) |
1214 | 1216 |
1215 # Since generaldelta is directly supported by cg2, reordering | 1217 # Since generaldelta is directly supported by cg2, reordering |
1216 # generally doesn't help, so we disable it by default (treating | 1218 # generally doesn't help, so we disable it by default (treating |
1220 allowreorder=False, | 1222 allowreorder=False, |
1221 builddeltaheader=builddeltaheader, | 1223 builddeltaheader=builddeltaheader, |
1222 manifestsend=b'', | 1224 manifestsend=b'', |
1223 sendtreemanifests=False, | 1225 sendtreemanifests=False, |
1224 bundlecaps=bundlecaps, | 1226 bundlecaps=bundlecaps, |
1227 ellipses=ellipses, | |
1225 shallow=shallow, | 1228 shallow=shallow, |
1226 ellipsisroots=ellipsisroots) | 1229 ellipsisroots=ellipsisroots) |
1227 | 1230 |
1228 def _makecg3packer(repo, filematcher, bundlecaps, shallow=False, | 1231 def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False, |
1229 ellipsisroots=None): | 1232 shallow=False, ellipsisroots=None): |
1230 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( | 1233 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( |
1231 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) | 1234 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) |
1232 | 1235 |
1233 return cgpacker(repo, filematcher, b'03', | 1236 return cgpacker(repo, filematcher, b'03', |
1234 useprevdelta=False, | 1237 useprevdelta=False, |
1235 allowreorder=False, | 1238 allowreorder=False, |
1236 builddeltaheader=builddeltaheader, | 1239 builddeltaheader=builddeltaheader, |
1237 manifestsend=closechunk(), | 1240 manifestsend=closechunk(), |
1238 sendtreemanifests=True, | 1241 sendtreemanifests=True, |
1239 bundlecaps=bundlecaps, | 1242 bundlecaps=bundlecaps, |
1243 ellipses=ellipses, | |
1240 shallow=shallow, | 1244 shallow=shallow, |
1241 ellipsisroots=ellipsisroots) | 1245 ellipsisroots=ellipsisroots) |
1242 | 1246 |
1243 _packermap = {'01': (_makecg1packer, cg1unpacker), | 1247 _packermap = {'01': (_makecg1packer, cg1unpacker), |
1244 # cg2 adds support for exchanging generaldelta | 1248 # cg2 adds support for exchanging generaldelta |
1297 versions.discard('01') | 1301 versions.discard('01') |
1298 assert versions | 1302 assert versions |
1299 return min(versions) | 1303 return min(versions) |
1300 | 1304 |
1301 def getbundler(version, repo, bundlecaps=None, filematcher=None, | 1305 def getbundler(version, repo, bundlecaps=None, filematcher=None, |
1302 shallow=False, ellipsisroots=None): | 1306 ellipses=False, shallow=False, ellipsisroots=None): |
1303 assert version in supportedoutgoingversions(repo) | 1307 assert version in supportedoutgoingversions(repo) |
1304 | 1308 |
1305 if filematcher is None: | 1309 if filematcher is None: |
1306 filematcher = matchmod.alwaysmatcher(repo.root, '') | 1310 filematcher = matchmod.alwaysmatcher(repo.root, '') |
1307 | 1311 |
1308 if version == '01' and not filematcher.always(): | 1312 if version == '01' and not filematcher.always(): |
1309 raise error.ProgrammingError('version 01 changegroups do not support ' | 1313 raise error.ProgrammingError('version 01 changegroups do not support ' |
1310 'sparse file matchers') | 1314 'sparse file matchers') |
1315 | |
1316 if ellipses and version in (b'01', b'02'): | |
1317 raise error.Abort( | |
1318 _('ellipsis nodes require at least cg3 on client and server, ' | |
1319 'but negotiated version %s') % version) | |
1311 | 1320 |
1312 # Requested files could include files not in the local store. So | 1321 # Requested files could include files not in the local store. So |
1313 # filter those out. | 1322 # filter those out. |
1314 filematcher = matchmod.intersectmatchers(repo.narrowmatch(), | 1323 filematcher = matchmod.intersectmatchers(repo.narrowmatch(), |
1315 filematcher) | 1324 filematcher) |
1316 | 1325 |
1317 fn = _packermap[version][0] | 1326 fn = _packermap[version][0] |
1318 return fn(repo, filematcher, bundlecaps, shallow=shallow, | 1327 return fn(repo, filematcher, bundlecaps, ellipses=ellipses, |
1319 ellipsisroots=ellipsisroots) | 1328 shallow=shallow, ellipsisroots=ellipsisroots) |
1320 | 1329 |
1321 def getunbundler(version, fh, alg, extras=None): | 1330 def getunbundler(version, fh, alg, extras=None): |
1322 return _packermap[version][1](fh, alg, extras=extras) | 1331 return _packermap[version][1](fh, alg, extras=extras) |
1323 | 1332 |
1324 def _changegroupinfo(repo, nodes, source): | 1333 def _changegroupinfo(repo, nodes, source): |
1400 | 1409 |
1401 return revisions, files | 1410 return revisions, files |
1402 | 1411 |
1403 def _packellipsischangegroup(repo, common, match, relevant_nodes, | 1412 def _packellipsischangegroup(repo, common, match, relevant_nodes, |
1404 ellipsisroots, visitnodes, depth, source, version): | 1413 ellipsisroots, visitnodes, depth, source, version): |
1405 if version in ('01', '02'): | |
1406 raise error.Abort( | |
1407 'ellipsis nodes require at least cg3 on client and server, ' | |
1408 'but negotiated version %s' % version) | |
1409 # We wrap cg1packer.revchunk, using a side channel to pass | 1414 # We wrap cg1packer.revchunk, using a side channel to pass |
1410 # relevant_nodes into that area. Then if linknode isn't in the | 1415 # relevant_nodes into that area. Then if linknode isn't in the |
1411 # set, we know we have an ellipsis node and we should defer | 1416 # set, we know we have an ellipsis node and we should defer |
1412 # sending that node's data. We override close() to detect | 1417 # sending that node's data. We override close() to detect |
1413 # pending ellipsis nodes and flush them. | 1418 # pending ellipsis nodes and flush them. |
1414 packer = getbundler(version, repo, filematcher=match, | 1419 packer = getbundler(version, repo, filematcher=match, |
1420 ellipses=True, | |
1415 shallow=depth is not None, | 1421 shallow=depth is not None, |
1416 ellipsisroots=ellipsisroots) | 1422 ellipsisroots=ellipsisroots) |
1417 # Give the packer the list of nodes which should not be | 1423 # Give the packer the list of nodes which should not be |
1418 # ellipsis nodes. We store this rather than the set of nodes | 1424 # ellipsis nodes. We store this rather than the set of nodes |
1419 # that should be an ellipsis because for very large histories | 1425 # that should be an ellipsis because for very large histories |