# HG changeset patch # User Gregory Szorc # Date 1468793631 25200 # Node ID 953839de96ab574caa40557c542c262286c6287c # Parent 37cccad5541005624274394f6247cfda75cd2b18 bundle2: store changeset count when creating file bundles The bundle2 changegroup part has an advisory param saying how many changesets are in the part. Before this patch, we were setting this part when generating bundle2 parts via the wire protocol but not when generating local bundle2 files. A side effect of not setting the changeset count part is that progress bars don't work when applying changesets. As the tests show, this impacted clone bundles, shelve, backup bundles, `hg unbundle`, and anything touching bundle2 files. This patch adds a backdoor to allow us to pass state from changegroup generation into the unbundler. We store the number of changesets in the changegroup in this state and use it to populate the aforementioned advisory part parameter when generating the bundle2 bundle. I concede that I'm not thrilled by how state is being passed in changegroup.py (it feels a bit hacky). I would love to overhaul the rather confusing set of functions in changegroup.py with something that passes rich objects around instead of e.g. low-level generators. However, given the code freeze for 3.9 is imminent, I'd rather not undertake this endeavor right now. This feels like the easiest way to get the parameter added to the changegroup part. diff -r 37cccad55410 -r 953839de96ab mercurial/bundle2.py --- a/mercurial/bundle2.py Sun Jul 17 15:10:30 2016 -0700 +++ b/mercurial/bundle2.py Sun Jul 17 15:13:51 2016 -0700 @@ -1294,6 +1294,9 @@ bundle.setcompression(compression) part = bundle.newpart('changegroup', data=cg.getchunks()) part.addparam('version', cg.version) + if 'clcount' in cg.extras: + part.addparam('nbchanges', str(cg.extras['clcount']), + mandatory=False) chunkiter = bundle.getchunks() else: # compression argument is only for the bundle2 case diff -r 37cccad55410 -r 953839de96ab mercurial/changegroup.py --- a/mercurial/changegroup.py Sun Jul 17 15:10:30 2016 -0700 +++ b/mercurial/changegroup.py Sun Jul 17 15:13:51 2016 -0700 @@ -135,7 +135,7 @@ version = '01' _grouplistcount = 1 # One list of files after the manifests - def __init__(self, fh, alg): + def __init__(self, fh, alg, extras=None): if alg == 'UN': alg = None # get more modern without breaking too much if not alg in util.decompressors: @@ -145,6 +145,7 @@ alg = '_truncatedBZ' self._stream = util.decompressors[alg](fh) self._type = alg + self.extras = extras or {} self.callback = None # These methods (compressed, read, seek, tell) all appear to only @@ -900,8 +901,8 @@ assert version in supportedoutgoingversions(repo) return _packermap[version][0](repo, bundlecaps) -def getunbundler(version, fh, alg): - return _packermap[version][1](fh, alg) +def getunbundler(version, fh, alg, extras=None): + return _packermap[version][1](fh, alg, extras=extras) def _changegroupinfo(repo, nodes, source): if repo.ui.verbose or source == 'bundle': @@ -929,7 +930,8 @@ def getsubset(repo, outgoing, bundler, source, fastpath=False): gengroup = getsubsetraw(repo, outgoing, bundler, source, fastpath) - return getunbundler(bundler.version, util.chunkbuffer(gengroup), None) + return getunbundler(bundler.version, util.chunkbuffer(gengroup), None, + {'clcount': len(outgoing.missing)}) def changegroupsubset(repo, roots, heads, source, version='01'): """Compute a changegroup consisting of all the nodes that are diff -r 37cccad55410 -r 953839de96ab tests/test-bundle-type.t --- a/tests/test-bundle-type.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-bundle-type.t Sun Jul 17 15:13:51 2016 -0700 @@ -52,7 +52,7 @@ 1 changesets found HG20\x00\x00 (esc) Stream params: {} - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf none-v2 @@ -61,7 +61,7 @@ 1 changesets found HG20\x00\x00 (esc) Stream params: sortdict([('Compression', 'BZ')]) - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf bzip2-v2 @@ -70,7 +70,7 @@ 1 changesets found HG20\x00\x00 (esc) Stream params: sortdict([('Compression', 'GZ')]) - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf gzip-v2 @@ -79,7 +79,7 @@ 1 changesets found HG20\x00\x00 (esc) Stream params: {} - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf none-v2 @@ -88,7 +88,7 @@ 1 changesets found HG20\x00\x00 (esc) Stream params: sortdict([('Compression', 'BZ')]) - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf bzip2-v2 diff -r 37cccad55410 -r 953839de96ab tests/test-bundle.t --- a/tests/test-bundle.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-bundle.t Sun Jul 17 15:13:51 2016 -0700 @@ -708,7 +708,7 @@ 1a38c1b849e8b70c756d2d80b0b9a3ac0b7ea11a 057f4db07f61970e1c11e83be79e9d08adc4dc31 bundle2-output-bundle: "HG20", (1 params) 1 parts total - bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload + bundle2-output-part: "changegroup" (params: 1 mandatory 1 advisory) streamed payload bundling: 1/2 changesets (50.00%) bundling: 2/2 changesets (100.00%) bundling: 1/2 manifests (50.00%) diff -r 37cccad55410 -r 953839de96ab tests/test-clonebundles.t --- a/tests/test-clonebundles.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-clonebundles.t Sun Jul 17 15:13:51 2016 -0700 @@ -156,33 +156,34 @@ by old clients. $ f --size --hexdump full.hg - full.hg: size=406 + full.hg: size=418 0000: 48 47 32 30 00 00 00 0e 43 6f 6d 70 72 65 73 73 |HG20....Compress| - 0010: 69 6f 6e 3d 47 5a 78 9c 63 60 60 90 e5 76 f6 70 |ion=GZx.c``..v.p| - 0020: f4 73 77 75 0f f2 0f 0d 60 00 02 46 06 76 a6 b2 |.swu....`..F.v..| - 0030: d4 a2 e2 cc fc 3c 03 23 06 06 e6 65 40 b1 4d c1 |.....<.#...e@.M.| - 0040: 2a 31 09 cf 9a 3a 52 04 b7 fc db f0 95 e5 a4 f4 |*1...:R.........| - 0050: 97 17 b2 c9 0c 14 00 02 e6 d9 99 25 1a a7 a4 99 |...........%....| - 0060: a4 a4 1a 5b 58 a4 19 27 9b a4 59 a4 1a 59 a4 99 |...[X..'..Y..Y..| - 0070: a4 59 26 5a 18 9a 18 59 5a 26 1a 27 27 25 99 a6 |.Y&Z...YZ&.''%..| - 0080: 99 1a 70 95 a4 16 97 70 19 28 18 70 a5 e5 e7 73 |..p....p.(.p...s| - 0090: 71 25 a6 a4 28 00 19 40 13 0e ac fa df ab ff 7b |q%..(..@.......{| - 00a0: 3f fb 92 dc 8b 1f 62 bb 9e b7 d7 d9 87 3d 5a 44 |?.....b......=ZD| - 00b0: ac 2f b0 a9 c3 66 1e 54 b9 26 08 a7 1a 1b 1a a7 |./...f.T.&......| - 00c0: 25 1b 9a 1b 99 19 9a 5a 18 9b a6 18 19 00 dd 67 |%......Z.......g| - 00d0: 61 61 98 06 f4 80 49 4a 8a 65 52 92 41 9a 81 81 |aa....IJ.eR.A...| - 00e0: a5 11 17 50 31 30 58 19 cc 80 98 25 29 b1 08 c4 |...P10X....%)...| - 00f0: 37 07 79 19 88 d9 41 ee 07 8a 41 cd 5d 98 65 fb |7.y...A...A.].e.| - 0100: e5 9e 45 bf 8d 7f 9f c6 97 9f 2b 44 34 67 d9 ec |..E.......+D4g..| - 0110: 8e 0f a0 61 a8 eb 82 82 2e c9 c2 20 25 d5 34 c5 |...a....... %.4.| - 0120: d0 d8 c2 dc d4 c2 d4 c4 30 d9 34 cd c0 d4 c8 cc |........0.4.....| - 0130: 34 31 c5 d0 c4 24 31 c9 32 2d d1 c2 2c c5 30 25 |41...$1.2-..,.0%| - 0140: 09 e4 ee 85 8f 85 ff 88 ab 89 36 c7 2a c4 47 34 |..........6.*.G4| - 0150: fe f8 ec 7b 73 37 3f c3 24 62 1d 8d 4d 1d 9e 40 |...{s7?.$b..M..@| - 0160: 06 3b 10 14 36 a4 38 10 04 d8 21 01 9a b1 83 f7 |.;..6.8...!.....| - 0170: e9 45 8b d2 56 c7 a3 1f 82 52 d7 8a 78 ed fc d5 |.E..V....R..x...| - 0180: 76 f1 36 25 81 89 c7 ad ec 90 34 48 75 2b 89 49 |v.6%......4Hu+.I| - 0190: bf 00 d6 97 f0 8d |......| + 0010: 69 6f 6e 3d 47 5a 78 9c 63 60 60 d0 e4 76 f6 70 |ion=GZx.c``..v.p| + 0020: f4 73 77 75 0f f2 0f 0d 60 00 02 46 46 76 26 4e |.swu....`..FFv&N| + 0030: c6 b2 d4 a2 e2 cc fc 3c 03 a3 bc a4 e4 8c c4 bc |.......<........| + 0040: f4 d4 62 23 06 06 e6 65 40 f9 4d c1 2a 31 09 cf |..b#...e@.M.*1..| + 0050: 9a 3a 52 04 b7 fc db f0 95 e5 a4 f4 97 17 b2 c9 |.:R.............| + 0060: 0c 14 00 02 e6 d9 99 25 1a a7 a4 99 a4 a4 1a 5b |.......%.......[| + 0070: 58 a4 19 27 9b a4 59 a4 1a 59 a4 99 a4 59 26 5a |X..'..Y..Y...Y&Z| + 0080: 18 9a 18 59 5a 26 1a 27 27 25 99 a6 99 1a 70 95 |...YZ&.''%....p.| + 0090: a4 16 97 70 19 28 18 70 a5 e5 e7 73 71 25 a6 a4 |...p.(.p...sq%..| + 00a0: 28 00 19 40 13 0e ac fa df ab ff 7b 3f fb 92 dc |(..@.......{?...| + 00b0: 8b 1f 62 bb 9e b7 d7 d9 87 3d 5a 44 ac 2f b0 a9 |..b......=ZD./..| + 00c0: c3 66 1e 54 b9 26 08 a7 1a 1b 1a a7 25 1b 9a 1b |.f.T.&......%...| + 00d0: 99 19 9a 5a 18 9b a6 18 19 00 dd 67 61 61 98 06 |...Z.......gaa..| + 00e0: f4 80 49 4a 8a 65 52 92 41 9a 81 81 a5 11 17 50 |..IJ.eR.A......P| + 00f0: 31 30 58 19 cc 80 98 25 29 b1 08 c4 37 07 79 19 |10X....%)...7.y.| + 0100: 88 d9 41 ee 07 8a 41 cd 5d 98 65 fb e5 9e 45 bf |..A...A.].e...E.| + 0110: 8d 7f 9f c6 97 9f 2b 44 34 67 d9 ec 8e 0f a0 61 |......+D4g.....a| + 0120: a8 eb 82 82 2e c9 c2 20 25 d5 34 c5 d0 d8 c2 dc |....... %.4.....| + 0130: d4 c2 d4 c4 30 d9 34 cd c0 d4 c8 cc 34 31 c5 d0 |....0.4.....41..| + 0140: c4 24 31 c9 32 2d d1 c2 2c c5 30 25 09 e4 ee 85 |.$1.2-..,.0%....| + 0150: 8f 85 ff 88 ab 89 36 c7 2a c4 47 34 fe f8 ec 7b |......6.*.G4...{| + 0160: 73 37 3f c3 24 62 1d 8d 4d 1d 9e 40 06 3b 10 14 |s7?.$b..M..@.;..| + 0170: 36 a4 38 10 04 d8 21 01 9a b1 83 f7 e9 45 8b d2 |6.8...!......E..| + 0180: 56 c7 a3 1f 82 52 d7 8a 78 ed fc d5 76 f1 36 25 |V....R..x...v.6%| + 0190: 81 89 c7 ad ec 90 34 48 75 2b 89 49 bf 00 cf 72 |......4Hu+.I...r| + 01a0: f4 7f |..| $ echo "http://localhost:$HGPORT1/full.hg" > server/.hg/clonebundles.manifest $ hg clone -U http://localhost:$HGPORT full-bundle diff -r 37cccad55410 -r 953839de96ab tests/test-debugbundle.t --- a/tests/test-debugbundle.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-debugbundle.t Sun Jul 17 15:13:51 2016 -0700 @@ -31,7 +31,7 @@ $ hg debugbundle bundle2.hg Stream params: {} - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '2')])" 0e067c57feba1a5694ca4844f05588bb1bf82342 991a3460af53952d10ec8a295d3d2cc2e5fa9690 @@ -56,7 +56,7 @@ $ hg debugbundle --all bundle2.hg Stream params: {} - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '2')])" format: id, p1, p2, cset, delta base, len(delta) changelog diff -r 37cccad55410 -r 953839de96ab tests/test-generaldelta.t --- a/tests/test-generaldelta.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-generaldelta.t Sun Jul 17 15:13:51 2016 -0700 @@ -155,7 +155,7 @@ saved backup bundle to $TESTTMP/aggressive/.hg/strip-backup/1c5d4dc9a8b8-6c68e60c-backup.hg (glob) $ hg debugbundle .hg/strip-backup/* Stream params: sortdict([('Compression', 'BZ')]) - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" 1c5d4dc9a8b8d6e1750966d343e94db665e7a1e9 $ cd .. diff -r 37cccad55410 -r 953839de96ab tests/test-getbundle.t --- a/tests/test-getbundle.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-getbundle.t Sun Jul 17 15:13:51 2016 -0700 @@ -170,7 +170,7 @@ $ hg debuggetbundle repo bundle -t bundle2 $ hg debugbundle bundle Stream params: {} - changegroup -- "sortdict([('version', '01')])" + changegroup -- "sortdict([('version', '01'), ('nbchanges', '18')])" 7704483d56b2a7b5db54dcee7c62378ac629b348 29a4d1f17bd3f0779ca0525bebb1cfb51067c738 713346a995c363120712aed1aee7e04afd867638 diff -r 37cccad55410 -r 953839de96ab tests/test-patchbomb.t --- a/tests/test-patchbomb.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-patchbomb.t Sun Jul 17 15:13:51 2016 -0700 @@ -352,14 +352,14 @@ Content-Disposition: attachment; filename="bundle.hg" Content-Transfer-Encoding: base64 - SEcyMAAAAA5Db21wcmVzc2lvbj1CWkJaaDkxQVkmU1lCZFwPAAAKf//7nFYSWD1/4H7R09C/I70I - Ak0E4peoSIYIgQCgGUQOcLABGY2hqoAAAaBMTTAAAahgTCZoAAAAAMQaqn5GmapojQ00DEGI/VGJ - kDAJoGTDUAAyM0QaAEqalPTUaMhoyDIDR6IxAGEGmgAehMRhDRsoyB6TYTC8JyLN+jTGqitRAgRJ - b3SRlhd8/+VxlAUqAilLoKPEEyxFQkaEGo+DzItFeNiFAo8NMMweVtvXJFIMhjoKC18DeYwjLKBz - wrMcs86qJrctDNJorwBMuLcqvTVWHh1IlsIaaaYSUIP2IZsogT1+pSSZS+bSTJrgfKsO9go/f0HF - uW4Yr2vXpxDreOgSIAdK/xC8Yay48SLpxIuqc/BZ6rVZCgG21rr0zhCaEgXOTqNaYEvANvg0B0Qo - dgtqAs1FDcZgzYitwJh6ZAG0C4mA7FPrp9b7h0h/A44Xgd+0it1gvF0mFE/CCPwymXS+OisOOCAF - mDUDAC1pBvsXckU4UJBCZFwP + SEcyMAAAAA5Db21wcmVzc2lvbj1CWkJaaDkxQVkmU1kIqE7KAAAKf//7vFYQWD1/4H7R09C/470I + Ak0E4peoSIYIgQCgGUQOcLABGY2hqoTTCYaBqaYAAACaaMAATIwAA1MIYDaQaqn6p+jRop+oJkA2 + oNqD1PU0PUBoxqaMmjMUepoBoDT1GmQNBKmlTT1GTCNMEAYTQ0NNDI0BoMQHpAZAA8o2pkyNJHfX + RRbXoyxKRUlAg41B3lpmMOnr77dEpFKAvEUGEkWuC4wioiMjC2Y2a84EXhsNCFIrbXUGId07PJnS + ELAOIpL/gE8R8CUeXuw2NKMtkFoLPkcTSomXtgHSg1IKaCNlWwVU3CpmMYqh5gkFYJKOD4UhVVQ6 + SiF1DpE8ghWvF1ih+fYgagfYHI96w/QsrRATpYiP7VRbINFrQy2c21mZ7M4pXXrPBypoXAIhtum7 + aKDJCpUqMDF5dfiDChMfgH9nQ4B60Uvgb4AK9dsbSYc+O3tEyNq9g9gZeA5Je2T82GzjC4DbY4F2 + 0kdrTBwslErFshCgDzeEBwICg13oQaQawQA1WWd3F3JFOFCQCKhOyg== --===============*==-- (glob) with a specific bundle type diff -r 37cccad55410 -r 953839de96ab tests/test-rebase-conflicts.t --- a/tests/test-rebase-conflicts.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-rebase-conflicts.t Sun Jul 17 15:13:51 2016 -0700 @@ -279,7 +279,7 @@ e31216eec445e44352c5f01588856059466a24c9 2f2496ddf49d69b5ef23ad8cf9fb2e0e4faf0ac2 bundle2-output-bundle: "HG20", (1 params) 1 parts total - bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload + bundle2-output-part: "changegroup" (params: 1 mandatory 1 advisory) streamed payload saved backup bundle to $TESTTMP/issue4041/.hg/strip-backup/e31216eec445-15f7a814-backup.hg (glob) 3 changesets found list of changesets: @@ -287,10 +287,10 @@ 19c888675e133ab5dff84516926a65672eaf04d9 2a7f09cac94c7f4b73ebd5cd1a62d3b2e8e336bf bundle2-output-bundle: "HG20", 1 parts total - bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload + bundle2-output-part: "changegroup" (params: 1 mandatory 1 advisory) streamed payload adding branch bundle2-input-bundle: with-transaction - bundle2-input-part: "changegroup" (params: 1 mandatory) supported + bundle2-input-part: "changegroup" (params: 1 mandatory 1 advisory) supported adding changesets add changeset 4c9fbe56a16f add changeset 19c888675e13 diff -r 37cccad55410 -r 953839de96ab tests/test-shelve.t --- a/tests/test-shelve.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-shelve.t Sun Jul 17 15:13:51 2016 -0700 @@ -1045,7 +1045,7 @@ 0 files updated, 0 files merged, 1 files removed, 0 files unresolved $ hg debugbundle .hg/shelved/*.hg Stream params: sortdict([('Compression', 'BZ')]) - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" 45993d65fe9dc3c6d8764b9c3b07fa831ee7d92d $ cd .. diff -r 37cccad55410 -r 953839de96ab tests/test-strip.t --- a/tests/test-strip.t Sun Jul 17 15:10:30 2016 -0700 +++ b/tests/test-strip.t Sun Jul 17 15:13:51 2016 -0700 @@ -211,7 +211,7 @@ $ hg debugbundle .hg/strip-backup/* Stream params: sortdict([('Compression', 'BZ')]) - changegroup -- "sortdict([('version', '02')])" + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" 264128213d290d868c54642d13aeaa3675551a78 $ hg pull .hg/strip-backup/* pulling from .hg/strip-backup/264128213d29-0b39d6bf-backup.hg @@ -799,7 +799,7 @@ 6625a516847449b6f0fa3737b9ba56e9f0f3032c d8db9d1372214336d2b5570f20ee468d2c72fa8b bundle2-output-bundle: "HG20", (1 params) 1 parts total - bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload + bundle2-output-part: "changegroup" (params: 1 mandatory 1 advisory) streamed payload saved backup bundle to $TESTTMP/issue4736/.hg/strip-backup/6625a5168474-345bb43d-backup.hg (glob) invalid branchheads cache (served): tip differs truncating cache/rbc-revs-v1 to 24