32 ) |
32 ) |
33 |
33 |
34 from .interfaces import repository |
34 from .interfaces import repository |
35 from .revlogutils import sidedata as sidedatamod |
35 from .revlogutils import sidedata as sidedatamod |
36 from .revlogutils import constants as revlog_constants |
36 from .revlogutils import constants as revlog_constants |
|
37 from .utils import storageutil |
37 |
38 |
38 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s") |
39 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s") |
39 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s") |
40 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s") |
40 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH") |
41 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH") |
|
42 _CHANGEGROUPV4_DELTA_HEADER = struct.Struct(b">B20s20s20s20s20sH") |
41 |
43 |
42 LFS_REQUIREMENT = b'lfs' |
44 LFS_REQUIREMENT = b'lfs' |
43 |
45 |
44 readexactly = util.readexactly |
46 readexactly = util.readexactly |
45 |
47 |
192 if prevnode is None: |
194 if prevnode is None: |
193 deltabase = p1 |
195 deltabase = p1 |
194 else: |
196 else: |
195 deltabase = prevnode |
197 deltabase = prevnode |
196 flags = 0 |
198 flags = 0 |
197 return node, p1, p2, deltabase, cs, flags |
199 protocol_flags = 0 |
|
200 return node, p1, p2, deltabase, cs, flags, protocol_flags |
198 |
201 |
199 def deltachunk(self, prevnode): |
202 def deltachunk(self, prevnode): |
200 l = self._chunklength() |
203 l = self._chunklength() |
201 if not l: |
204 if not l: |
202 return {} |
205 return {} |
203 headerdata = readexactly(self._stream, self.deltaheadersize) |
206 headerdata = readexactly(self._stream, self.deltaheadersize) |
204 header = self.deltaheader.unpack(headerdata) |
207 header = self.deltaheader.unpack(headerdata) |
205 delta = readexactly(self._stream, l - self.deltaheadersize) |
208 delta = readexactly(self._stream, l - self.deltaheadersize) |
206 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) |
209 header = self._deltaheader(header, prevnode) |
207 # cg4 forward-compat |
210 node, p1, p2, deltabase, cs, flags, protocol_flags = header |
208 sidedata = {} |
211 return node, p1, p2, cs, deltabase, delta, flags, protocol_flags |
209 return (node, p1, p2, cs, deltabase, delta, flags, sidedata) |
|
210 |
212 |
211 def getchunks(self): |
213 def getchunks(self): |
212 """returns all the chunks contains in the bundle |
214 """returns all the chunks contains in the bundle |
213 |
215 |
214 Used when you need to forward the binary stream to a file or another |
216 Used when you need to forward the binary stream to a file or another |
595 version = b'02' |
597 version = b'02' |
596 |
598 |
597 def _deltaheader(self, headertuple, prevnode): |
599 def _deltaheader(self, headertuple, prevnode): |
598 node, p1, p2, deltabase, cs = headertuple |
600 node, p1, p2, deltabase, cs = headertuple |
599 flags = 0 |
601 flags = 0 |
600 return node, p1, p2, deltabase, cs, flags |
602 protocol_flags = 0 |
|
603 return node, p1, p2, deltabase, cs, flags, protocol_flags |
601 |
604 |
602 |
605 |
603 class cg3unpacker(cg2unpacker): |
606 class cg3unpacker(cg2unpacker): |
604 """Unpacker for cg3 streams. |
607 """Unpacker for cg3 streams. |
605 |
608 |
613 version = b'03' |
616 version = b'03' |
614 _grouplistcount = 2 # One list of manifests and one list of files |
617 _grouplistcount = 2 # One list of manifests and one list of files |
615 |
618 |
616 def _deltaheader(self, headertuple, prevnode): |
619 def _deltaheader(self, headertuple, prevnode): |
617 node, p1, p2, deltabase, cs, flags = headertuple |
620 node, p1, p2, deltabase, cs, flags = headertuple |
618 return node, p1, p2, deltabase, cs, flags |
621 protocol_flags = 0 |
|
622 return node, p1, p2, deltabase, cs, flags, protocol_flags |
619 |
623 |
620 def _unpackmanifests(self, repo, revmap, trp, prog, addrevisioncb=None): |
624 def _unpackmanifests(self, repo, revmap, trp, prog, addrevisioncb=None): |
621 super(cg3unpacker, self)._unpackmanifests( |
625 super(cg3unpacker, self)._unpackmanifests( |
622 repo, revmap, trp, prog, addrevisioncb=addrevisioncb |
626 repo, revmap, trp, prog, addrevisioncb=addrevisioncb |
623 ) |
627 ) |
636 """Unpacker for cg4 streams. |
640 """Unpacker for cg4 streams. |
637 |
641 |
638 cg4 streams add support for exchanging sidedata. |
642 cg4 streams add support for exchanging sidedata. |
639 """ |
643 """ |
640 |
644 |
|
645 deltaheader = _CHANGEGROUPV4_DELTA_HEADER |
|
646 deltaheadersize = deltaheader.size |
641 version = b'04' |
647 version = b'04' |
|
648 |
|
649 def _deltaheader(self, headertuple, prevnode): |
|
650 protocol_flags, node, p1, p2, deltabase, cs, flags = headertuple |
|
651 return node, p1, p2, deltabase, cs, flags, protocol_flags |
642 |
652 |
643 def deltachunk(self, prevnode): |
653 def deltachunk(self, prevnode): |
644 res = super(cg4unpacker, self).deltachunk(prevnode) |
654 res = super(cg4unpacker, self).deltachunk(prevnode) |
645 if not res: |
655 if not res: |
646 return res |
656 return res |
647 |
657 |
648 (node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res |
658 (node, p1, p2, cs, deltabase, delta, flags, protocol_flags) = res |
649 |
659 |
650 sidedata_raw = getchunk(self._stream) |
|
651 sidedata = {} |
660 sidedata = {} |
652 if len(sidedata_raw) > 0: |
661 if protocol_flags & storageutil.CG_FLAG_SIDEDATA: |
|
662 sidedata_raw = getchunk(self._stream) |
653 sidedata = sidedatamod.deserialize_sidedata(sidedata_raw) |
663 sidedata = sidedatamod.deserialize_sidedata(sidedata_raw) |
654 |
664 |
655 return node, p1, p2, cs, deltabase, delta, flags, sidedata |
665 return node, p1, p2, cs, deltabase, delta, flags, sidedata |
656 |
666 |
657 |
667 |
693 yield meta |
703 yield meta |
694 if prefix: |
704 if prefix: |
695 yield prefix |
705 yield prefix |
696 yield data |
706 yield data |
697 |
707 |
698 sidedata = delta.sidedata |
708 if delta.protocol_flags & storageutil.CG_FLAG_SIDEDATA: |
699 if sidedata is not None: |
|
700 # Need a separate chunk for sidedata to be able to differentiate |
709 # Need a separate chunk for sidedata to be able to differentiate |
701 # "raw delta" length and sidedata length |
710 # "raw delta" length and sidedata length |
|
711 sidedata = delta.sidedata |
702 yield chunkheader(len(sidedata)) |
712 yield chunkheader(len(sidedata)) |
703 yield sidedata |
713 yield sidedata |
704 |
714 |
705 |
715 |
706 def _sortnodesellipsis(store, nodes, cl, lookup): |
716 def _sortnodesellipsis(store, nodes, cl, lookup): |
1638 shallow=False, |
1648 shallow=False, |
1639 ellipsisroots=None, |
1649 ellipsisroots=None, |
1640 fullnodes=None, |
1650 fullnodes=None, |
1641 remote_sidedata=None, |
1651 remote_sidedata=None, |
1642 ): |
1652 ): |
1643 # Same header func as cg3. Sidedata is in a separate chunk from the delta to |
1653 # Sidedata is in a separate chunk from the delta to differentiate |
1644 # differenciate "raw delta" and sidedata. |
1654 # "raw delta" and sidedata. |
1645 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( |
1655 def builddeltaheader(d): |
1646 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags |
1656 return _CHANGEGROUPV4_DELTA_HEADER.pack( |
1647 ) |
1657 d.protocol_flags, |
|
1658 d.node, |
|
1659 d.p1node, |
|
1660 d.p2node, |
|
1661 d.basenode, |
|
1662 d.linknode, |
|
1663 d.flags, |
|
1664 ) |
1648 |
1665 |
1649 return cgpacker( |
1666 return cgpacker( |
1650 repo, |
1667 repo, |
1651 oldmatcher, |
1668 oldmatcher, |
1652 matcher, |
1669 matcher, |
1928 def get_sidedata_helpers(repo, remote_sd_categories, pull=False): |
1945 def get_sidedata_helpers(repo, remote_sd_categories, pull=False): |
1929 # Computers for computing sidedata on-the-fly |
1946 # Computers for computing sidedata on-the-fly |
1930 sd_computers = collections.defaultdict(list) |
1947 sd_computers = collections.defaultdict(list) |
1931 # Computers for categories to remove from sidedata |
1948 # Computers for categories to remove from sidedata |
1932 sd_removers = collections.defaultdict(list) |
1949 sd_removers = collections.defaultdict(list) |
1933 |
|
1934 to_generate = remote_sd_categories - repo._wanted_sidedata |
1950 to_generate = remote_sd_categories - repo._wanted_sidedata |
1935 to_remove = repo._wanted_sidedata - remote_sd_categories |
1951 to_remove = repo._wanted_sidedata - remote_sd_categories |
1936 if pull: |
1952 if pull: |
1937 to_generate, to_remove = to_remove, to_generate |
1953 to_generate, to_remove = to_remove, to_generate |
1938 |
1954 |