Mercurial > hg
comparison mercurial/changegroup.py @ 39014:d662959dc881
changegroup: emit revisiondelta instances from deltagroup()
By abstracting the concept of a delta group away from its
serialization (the changegroup format), we make it easier
to establish alternate serialization formats. We also make
it possible to move aspects of delta group generation into
the storage layer. This will allow storage to make decisions
about e.g. delta parent choices without the changegroup code
needing storage APIs to determine delta parents. We're still
a bit of a way from there. Future commits will work towards
that world.
Differential Revision: https://phab.mercurial-scm.org/D4211
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Wed, 08 Aug 2018 14:44:48 -0700 |
parents | c4a2d19d393a |
children | ad9eccedb379 |
comparison
equal
deleted
inserted
replaced
39013:c4a2d19d393a | 39014:d662959dc881 |
---|---|
519 linknode = attr.ib() | 519 linknode = attr.ib() |
520 # 2 bytes of flags to apply to revision data. | 520 # 2 bytes of flags to apply to revision data. |
521 flags = attr.ib() | 521 flags = attr.ib() |
522 # Iterable of chunks holding raw delta data. | 522 # Iterable of chunks holding raw delta data. |
523 deltachunks = attr.ib() | 523 deltachunks = attr.ib() |
524 | |
525 def _revisiondeltatochunks(delta, headerfn): | |
526 """Serialize a revisiondelta to changegroup chunks.""" | |
527 meta = headerfn(delta) | |
528 l = len(meta) + sum(len(x) for x in delta.deltachunks) | |
529 yield chunkheader(l) | |
530 yield meta | |
531 for x in delta.deltachunks: | |
532 yield x | |
524 | 533 |
525 def _sortnodesnormal(store, nodes, reorder): | 534 def _sortnodesnormal(store, nodes, reorder): |
526 """Sort nodes for changegroup generation and turn into revnums.""" | 535 """Sort nodes for changegroup generation and turn into revnums.""" |
527 # for generaldelta revlogs, we linearize the revs; this will both be | 536 # for generaldelta revlogs, we linearize the revs; this will both be |
528 # much quicker and generate a much smaller bundle | 537 # much quicker and generate a much smaller bundle |
680 flags=flags, | 689 flags=flags, |
681 deltachunks=(diffheader, data), | 690 deltachunks=(diffheader, data), |
682 ) | 691 ) |
683 | 692 |
684 def deltagroup(repo, revs, store, ischangelog, lookup, deltaparentfn, | 693 def deltagroup(repo, revs, store, ischangelog, lookup, deltaparentfn, |
685 deltaheaderfn, units=None, | 694 units=None, |
686 ellipses=False, clrevtolocalrev=None, fullclnodes=None, | 695 ellipses=False, clrevtolocalrev=None, fullclnodes=None, |
687 precomputedellipsis=None): | 696 precomputedellipsis=None): |
688 """Calculate a delta group, yielding a sequence of changegroup chunks | 697 """Calculate deltas for a set of revisions. |
689 (strings). | 698 |
690 | 699 Is a generator of ``revisiondelta`` instances. |
691 Given a list of changeset revs, return a set of deltas and | |
692 metadata corresponding to nodes. The first delta is | |
693 first parent(nodelist[0]) -> nodelist[0], the receiver is | |
694 guaranteed to have this parent as it has all history before | |
695 these changesets. In the case firstparent is nullrev the | |
696 changegroup starts with a full revision. | |
697 | 700 |
698 If units is not None, progress detail will be generated, units specifies | 701 If units is not None, progress detail will be generated, units specifies |
699 the type of revlog that is touched (changelog, manifest, etc.). | 702 the type of revlog that is touched (changelog, manifest, etc.). |
700 """ | 703 """ |
701 # if we don't have any revisions touched by these changesets, bail | 704 # if we don't have any revisions touched by these changesets, bail |
737 precomputedellipsis) | 740 precomputedellipsis) |
738 else: | 741 else: |
739 delta = _revisiondeltanormal(store, curr, prev, linknode, | 742 delta = _revisiondeltanormal(store, curr, prev, linknode, |
740 deltaparentfn) | 743 deltaparentfn) |
741 | 744 |
742 if not delta: | 745 if delta: |
743 continue | 746 yield delta |
744 | |
745 meta = deltaheaderfn(delta) | |
746 l = len(meta) + sum(len(x) for x in delta.deltachunks) | |
747 yield chunkheader(l) | |
748 yield meta | |
749 for x in delta.deltachunks: | |
750 yield x | |
751 | 747 |
752 if progress: | 748 if progress: |
753 progress.complete() | 749 progress.complete() |
754 | 750 |
755 class cgpacker(object): | 751 class cgpacker(object): |
829 cl = repo.changelog | 825 cl = repo.changelog |
830 | 826 |
831 self._verbosenote(_('uncompressed size of bundle content:\n')) | 827 self._verbosenote(_('uncompressed size of bundle content:\n')) |
832 size = 0 | 828 size = 0 |
833 | 829 |
834 clstate, chunks = self._generatechangelog(cl, clnodes) | 830 clstate, deltas = self._generatechangelog(cl, clnodes) |
835 for chunk in chunks: | 831 for delta in deltas: |
836 size += len(chunk) | 832 for chunk in _revisiondeltatochunks(delta, self._builddeltaheader): |
837 yield chunk | 833 size += len(chunk) |
834 yield chunk | |
838 | 835 |
839 close = closechunk() | 836 close = closechunk() |
840 size += len(close) | 837 size += len(close) |
841 yield closechunk() | 838 yield closechunk() |
842 | 839 |
873 size = 0 | 870 size = 0 |
874 it = self.generatemanifests( | 871 it = self.generatemanifests( |
875 commonrevs, clrevorder, fastpathlinkrev, mfs, fnodes, source, | 872 commonrevs, clrevorder, fastpathlinkrev, mfs, fnodes, source, |
876 clstate['clrevtomanifestrev']) | 873 clstate['clrevtomanifestrev']) |
877 | 874 |
878 for dir, chunks in it: | 875 for dir, deltas in it: |
879 if dir: | 876 if dir: |
880 assert self.version == b'03' | 877 assert self.version == b'03' |
881 chunk = _fileheader(dir) | 878 chunk = _fileheader(dir) |
882 size += len(chunk) | 879 size += len(chunk) |
883 yield chunk | 880 yield chunk |
884 | 881 |
885 for chunk in chunks: | 882 for delta in deltas: |
886 size += len(chunk) | 883 chunks = _revisiondeltatochunks(delta, self._builddeltaheader) |
887 yield chunk | 884 for chunk in chunks: |
885 size += len(chunk) | |
886 yield chunk | |
888 | 887 |
889 close = closechunk() | 888 close = closechunk() |
890 size += len(close) | 889 size += len(close) |
891 yield close | 890 yield close |
892 | 891 |
903 | 902 |
904 it = self.generatefiles(changedfiles, commonrevs, | 903 it = self.generatefiles(changedfiles, commonrevs, |
905 source, mfdicts, fastpathlinkrev, | 904 source, mfdicts, fastpathlinkrev, |
906 fnodes, clrevs) | 905 fnodes, clrevs) |
907 | 906 |
908 for path, chunks in it: | 907 for path, deltas in it: |
909 h = _fileheader(path) | 908 h = _fileheader(path) |
910 size = len(h) | 909 size = len(h) |
911 yield h | 910 yield h |
912 | 911 |
913 for chunk in chunks: | 912 for delta in deltas: |
914 size += len(chunk) | 913 chunks = _revisiondeltatochunks(delta, self._builddeltaheader) |
915 yield chunk | 914 for chunk in chunks: |
915 size += len(chunk) | |
916 yield chunk | |
916 | 917 |
917 close = closechunk() | 918 close = closechunk() |
918 size += len(close) | 919 size += len(close) |
919 yield close | 920 yield close |
920 | 921 |
991 'clrevtomanifestrev': clrevtomanifestrev, | 992 'clrevtomanifestrev': clrevtomanifestrev, |
992 } | 993 } |
993 | 994 |
994 gen = deltagroup( | 995 gen = deltagroup( |
995 self._repo, revs, cl, True, lookupcl, | 996 self._repo, revs, cl, True, lookupcl, |
996 self._deltaparentfn, self._builddeltaheader, | 997 self._deltaparentfn, |
997 ellipses=self._ellipses, | 998 ellipses=self._ellipses, |
998 units=_('changesets'), | 999 units=_('changesets'), |
999 clrevtolocalrev={}, | 1000 clrevtolocalrev={}, |
1000 fullclnodes=self._fullclnodes, | 1001 fullclnodes=self._fullclnodes, |
1001 precomputedellipsis=self._precomputedellipsis) | 1002 precomputedellipsis=self._precomputedellipsis) |
1078 lookupfn) | 1079 lookupfn) |
1079 else: | 1080 else: |
1080 revs = _sortnodesnormal(store, prunednodes, | 1081 revs = _sortnodesnormal(store, prunednodes, |
1081 self._reorder) | 1082 self._reorder) |
1082 | 1083 |
1083 it = deltagroup( | 1084 deltas = deltagroup( |
1084 self._repo, revs, store, False, lookupfn, | 1085 self._repo, revs, store, False, lookupfn, |
1085 self._deltaparentfn, self._builddeltaheader, | 1086 self._deltaparentfn, |
1086 ellipses=self._ellipses, | 1087 ellipses=self._ellipses, |
1087 units=_('manifests'), | 1088 units=_('manifests'), |
1088 clrevtolocalrev=clrevtolocalrev, | 1089 clrevtolocalrev=clrevtolocalrev, |
1089 fullclnodes=self._fullclnodes, | 1090 fullclnodes=self._fullclnodes, |
1090 precomputedellipsis=self._precomputedellipsis) | 1091 precomputedellipsis=self._precomputedellipsis) |
1091 | 1092 |
1092 yield dir, it | 1093 yield dir, deltas |
1093 | 1094 |
1094 # The 'source' parameter is useful for extensions | 1095 # The 'source' parameter is useful for extensions |
1095 def generatefiles(self, changedfiles, commonrevs, source, | 1096 def generatefiles(self, changedfiles, commonrevs, source, |
1096 mfdicts, fastpathlinkrev, fnodes, clrevs): | 1097 mfdicts, fastpathlinkrev, fnodes, clrevs): |
1097 changedfiles = list(filter(self._filematcher, changedfiles)) | 1098 changedfiles = list(filter(self._filematcher, changedfiles)) |
1170 revs = _sortnodesnormal(filerevlog, filenodes, | 1171 revs = _sortnodesnormal(filerevlog, filenodes, |
1171 self._reorder) | 1172 self._reorder) |
1172 | 1173 |
1173 progress.update(i + 1, item=fname) | 1174 progress.update(i + 1, item=fname) |
1174 | 1175 |
1175 it = deltagroup( | 1176 deltas = deltagroup( |
1176 self._repo, revs, filerevlog, False, lookupfilelog, | 1177 self._repo, revs, filerevlog, False, lookupfilelog, |
1177 self._deltaparentfn, self._builddeltaheader, | 1178 self._deltaparentfn, |
1178 ellipses=self._ellipses, | 1179 ellipses=self._ellipses, |
1179 clrevtolocalrev=clrevtolocalrev, | 1180 clrevtolocalrev=clrevtolocalrev, |
1180 fullclnodes=self._fullclnodes, | 1181 fullclnodes=self._fullclnodes, |
1181 precomputedellipsis=self._precomputedellipsis) | 1182 precomputedellipsis=self._precomputedellipsis) |
1182 | 1183 |
1183 yield fname, it | 1184 yield fname, deltas |
1184 | 1185 |
1185 progress.complete() | 1186 progress.complete() |
1186 | 1187 |
1187 def _deltaparentprev(store, rev, p1, p2, prev): | 1188 def _deltaparentprev(store, rev, p1, p2, prev): |
1188 """Resolve a delta parent to the previous revision. | 1189 """Resolve a delta parent to the previous revision. |