comparison mercurial/changegroup.py @ 43077:687b865b95ad

formatting: byteify all mercurial/ and hgext/ string literals Done with python3.7 contrib/byteify-strings.py -i $(hg files 'set:mercurial/**.py - mercurial/thirdparty/** + hgext/**.py - hgext/fsmonitor/pywatchman/** - mercurial/__init__.py') black -l 80 -t py33 -S $(hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**" - hgext/fsmonitor/pywatchman/**') # skip-blame mass-reformatting only Differential Revision: https://phab.mercurial-scm.org/D6972
author Augie Fackler <augie@google.com>
date Sun, 06 Oct 2019 09:48:39 -0400
parents 2372284d9457
children eef9a2d67051
comparison
equal deleted inserted replaced
43076:2372284d9457 43077:687b865b95ad
28 util, 28 util,
29 ) 29 )
30 30
31 from .interfaces import repository 31 from .interfaces import repository
32 32
33 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s") 33 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s")
34 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s") 34 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s")
35 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH") 35 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH")
36 36
37 LFS_REQUIREMENT = 'lfs' 37 LFS_REQUIREMENT = b'lfs'
38 38
39 readexactly = util.readexactly 39 readexactly = util.readexactly
40 40
41 41
42 def getchunk(stream): 42 def getchunk(stream):
43 """return the next chunk from stream as a string""" 43 """return the next chunk from stream as a string"""
44 d = readexactly(stream, 4) 44 d = readexactly(stream, 4)
45 l = struct.unpack(">l", d)[0] 45 l = struct.unpack(b">l", d)[0]
46 if l <= 4: 46 if l <= 4:
47 if l: 47 if l:
48 raise error.Abort(_("invalid chunk length %d") % l) 48 raise error.Abort(_(b"invalid chunk length %d") % l)
49 return "" 49 return b""
50 return readexactly(stream, l - 4) 50 return readexactly(stream, l - 4)
51 51
52 52
53 def chunkheader(length): 53 def chunkheader(length):
54 """return a changegroup chunk header (string)""" 54 """return a changegroup chunk header (string)"""
55 return struct.pack(">l", length + 4) 55 return struct.pack(b">l", length + 4)
56 56
57 57
58 def closechunk(): 58 def closechunk():
59 """return a changegroup chunk header (string) for a zero-length chunk""" 59 """return a changegroup chunk header (string) for a zero-length chunk"""
60 return struct.pack(">l", 0) 60 return struct.pack(b">l", 0)
61 61
62 62
63 def _fileheader(path): 63 def _fileheader(path):
64 """Obtain a changegroup chunk header for a named path.""" 64 """Obtain a changegroup chunk header for a named path."""
65 return chunkheader(len(path)) + path 65 return chunkheader(len(path)) + path
75 fh = None 75 fh = None
76 cleanup = None 76 cleanup = None
77 try: 77 try:
78 if filename: 78 if filename:
79 if vfs: 79 if vfs:
80 fh = vfs.open(filename, "wb") 80 fh = vfs.open(filename, b"wb")
81 else: 81 else:
82 # Increase default buffer size because default is usually 82 # Increase default buffer size because default is usually
83 # small (4k is common on Linux). 83 # small (4k is common on Linux).
84 fh = open(filename, "wb", 131072) 84 fh = open(filename, b"wb", 131072)
85 else: 85 else:
86 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg") 86 fd, filename = pycompat.mkstemp(prefix=b"hg-bundle-", suffix=b".hg")
87 fh = os.fdopen(fd, r"wb") 87 fh = os.fdopen(fd, r"wb")
88 cleanup = filename 88 cleanup = filename
89 for c in chunks: 89 for c in chunks:
90 fh.write(c) 90 fh.write(c)
91 cleanup = None 91 cleanup = None
119 bundlerepo and some debug commands - their use is discouraged. 119 bundlerepo and some debug commands - their use is discouraged.
120 """ 120 """
121 121
122 deltaheader = _CHANGEGROUPV1_DELTA_HEADER 122 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
123 deltaheadersize = deltaheader.size 123 deltaheadersize = deltaheader.size
124 version = '01' 124 version = b'01'
125 _grouplistcount = 1 # One list of files after the manifests 125 _grouplistcount = 1 # One list of files after the manifests
126 126
127 def __init__(self, fh, alg, extras=None): 127 def __init__(self, fh, alg, extras=None):
128 if alg is None: 128 if alg is None:
129 alg = 'UN' 129 alg = b'UN'
130 if alg not in util.compengines.supportedbundletypes: 130 if alg not in util.compengines.supportedbundletypes:
131 raise error.Abort(_('unknown stream compression type: %s') % alg) 131 raise error.Abort(_(b'unknown stream compression type: %s') % alg)
132 if alg == 'BZ': 132 if alg == b'BZ':
133 alg = '_truncatedBZ' 133 alg = b'_truncatedBZ'
134 134
135 compengine = util.compengines.forbundletype(alg) 135 compengine = util.compengines.forbundletype(alg)
136 self._stream = compengine.decompressorreader(fh) 136 self._stream = compengine.decompressorreader(fh)
137 self._type = alg 137 self._type = alg
138 self.extras = extras or {} 138 self.extras = extras or {}
139 self.callback = None 139 self.callback = None
140 140
141 # These methods (compressed, read, seek, tell) all appear to only 141 # These methods (compressed, read, seek, tell) all appear to only
142 # be used by bundlerepo, but it's a little hard to tell. 142 # be used by bundlerepo, but it's a little hard to tell.
143 def compressed(self): 143 def compressed(self):
144 return self._type is not None and self._type != 'UN' 144 return self._type is not None and self._type != b'UN'
145 145
146 def read(self, l): 146 def read(self, l):
147 return self._stream.read(l) 147 return self._stream.read(l)
148 148
149 def seek(self, pos): 149 def seek(self, pos):
155 def close(self): 155 def close(self):
156 return self._stream.close() 156 return self._stream.close()
157 157
158 def _chunklength(self): 158 def _chunklength(self):
159 d = readexactly(self._stream, 4) 159 d = readexactly(self._stream, 4)
160 l = struct.unpack(">l", d)[0] 160 l = struct.unpack(b">l", d)[0]
161 if l <= 4: 161 if l <= 4:
162 if l: 162 if l:
163 raise error.Abort(_("invalid chunk length %d") % l) 163 raise error.Abort(_(b"invalid chunk length %d") % l)
164 return 0 164 return 0
165 if self.callback: 165 if self.callback:
166 self.callback() 166 self.callback()
167 return l - 4 167 return l - 4
168 168
178 """return the header of the filelogs chunk, v10 only has the filename""" 178 """return the header of the filelogs chunk, v10 only has the filename"""
179 l = self._chunklength() 179 l = self._chunklength()
180 if not l: 180 if not l:
181 return {} 181 return {}
182 fname = readexactly(self._stream, l) 182 fname = readexactly(self._stream, l)
183 return {'filename': fname} 183 return {b'filename': fname}
184 184
185 def _deltaheader(self, headertuple, prevnode): 185 def _deltaheader(self, headertuple, prevnode):
186 node, p1, p2, cs = headertuple 186 node, p1, p2, cs = headertuple
187 if prevnode is None: 187 if prevnode is None:
188 deltabase = p1 188 deltabase = p1
278 - number of heads stays the same: 1 278 - number of heads stays the same: 1
279 """ 279 """
280 repo = repo.unfiltered() 280 repo = repo.unfiltered()
281 281
282 def csmap(x): 282 def csmap(x):
283 repo.ui.debug("add changeset %s\n" % short(x)) 283 repo.ui.debug(b"add changeset %s\n" % short(x))
284 return len(cl) 284 return len(cl)
285 285
286 def revmap(x): 286 def revmap(x):
287 return cl.rev(x) 287 return cl.rev(x)
288 288
291 try: 291 try:
292 # The transaction may already carry source information. In this 292 # The transaction may already carry source information. In this
293 # case we use the top level data. We overwrite the argument 293 # case we use the top level data. We overwrite the argument
294 # because we need to use the top level value (if they exist) 294 # because we need to use the top level value (if they exist)
295 # in this function. 295 # in this function.
296 srctype = tr.hookargs.setdefault('source', srctype) 296 srctype = tr.hookargs.setdefault(b'source', srctype)
297 tr.hookargs.setdefault('url', url) 297 tr.hookargs.setdefault(b'url', url)
298 repo.hook( 298 repo.hook(
299 'prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs) 299 b'prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs)
300 ) 300 )
301 301
302 # write changelog data to temp files so concurrent readers 302 # write changelog data to temp files so concurrent readers
303 # will not see an inconsistent view 303 # will not see an inconsistent view
304 cl = repo.changelog 304 cl = repo.changelog
305 cl.delayupdate(tr) 305 cl.delayupdate(tr)
306 oldheads = set(cl.heads()) 306 oldheads = set(cl.heads())
307 307
308 trp = weakref.proxy(tr) 308 trp = weakref.proxy(tr)
309 # pull off the changeset group 309 # pull off the changeset group
310 repo.ui.status(_("adding changesets\n")) 310 repo.ui.status(_(b"adding changesets\n"))
311 clstart = len(cl) 311 clstart = len(cl)
312 progress = repo.ui.makeprogress( 312 progress = repo.ui.makeprogress(
313 _('changesets'), unit=_('chunks'), total=expectedtotal 313 _(b'changesets'), unit=_(b'chunks'), total=expectedtotal
314 ) 314 )
315 self.callback = progress.increment 315 self.callback = progress.increment
316 316
317 efiles = set() 317 efiles = set()
318 318
324 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog) 324 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
325 efiles = len(efiles) 325 efiles = len(efiles)
326 326
327 if not cgnodes: 327 if not cgnodes:
328 repo.ui.develwarn( 328 repo.ui.develwarn(
329 'applied empty changelog from changegroup', 329 b'applied empty changelog from changegroup',
330 config='warn-empty-changegroup', 330 config=b'warn-empty-changegroup',
331 ) 331 )
332 clend = len(cl) 332 clend = len(cl)
333 changesets = clend - clstart 333 changesets = clend - clstart
334 progress.complete() 334 progress.complete()
335 self.callback = None 335 self.callback = None
336 336
337 # pull off the manifest group 337 # pull off the manifest group
338 repo.ui.status(_("adding manifests\n")) 338 repo.ui.status(_(b"adding manifests\n"))
339 # We know that we'll never have more manifests than we had 339 # We know that we'll never have more manifests than we had
340 # changesets. 340 # changesets.
341 progress = repo.ui.makeprogress( 341 progress = repo.ui.makeprogress(
342 _('manifests'), unit=_('chunks'), total=changesets 342 _(b'manifests'), unit=_(b'chunks'), total=changesets
343 ) 343 )
344 self._unpackmanifests(repo, revmap, trp, progress) 344 self._unpackmanifests(repo, revmap, trp, progress)
345 345
346 needfiles = {} 346 needfiles = {}
347 if repo.ui.configbool('server', 'validate'): 347 if repo.ui.configbool(b'server', b'validate'):
348 cl = repo.changelog 348 cl = repo.changelog
349 ml = repo.manifestlog 349 ml = repo.manifestlog
350 # validate incoming csets have their manifests 350 # validate incoming csets have their manifests
351 for cset in pycompat.xrange(clstart, clend): 351 for cset in pycompat.xrange(clstart, clend):
352 mfnode = cl.changelogrevision(cset).manifest 352 mfnode = cl.changelogrevision(cset).manifest
354 # store file cgnodes we must see 354 # store file cgnodes we must see
355 for f, n in mfest.iteritems(): 355 for f, n in mfest.iteritems():
356 needfiles.setdefault(f, set()).add(n) 356 needfiles.setdefault(f, set()).add(n)
357 357
358 # process the files 358 # process the files
359 repo.ui.status(_("adding file changes\n")) 359 repo.ui.status(_(b"adding file changes\n"))
360 newrevs, newfiles = _addchangegroupfiles( 360 newrevs, newfiles = _addchangegroupfiles(
361 repo, self, revmap, trp, efiles, needfiles 361 repo, self, revmap, trp, efiles, needfiles
362 ) 362 )
363 363
364 # making sure the value exists 364 # making sure the value exists
365 tr.changes.setdefault('changegroup-count-changesets', 0) 365 tr.changes.setdefault(b'changegroup-count-changesets', 0)
366 tr.changes.setdefault('changegroup-count-revisions', 0) 366 tr.changes.setdefault(b'changegroup-count-revisions', 0)
367 tr.changes.setdefault('changegroup-count-files', 0) 367 tr.changes.setdefault(b'changegroup-count-files', 0)
368 tr.changes.setdefault('changegroup-count-heads', 0) 368 tr.changes.setdefault(b'changegroup-count-heads', 0)
369 369
370 # some code use bundle operation for internal purpose. They usually 370 # some code use bundle operation for internal purpose. They usually
371 # set `ui.quiet` to do this outside of user sight. Size the report 371 # set `ui.quiet` to do this outside of user sight. Size the report
372 # of such operation now happens at the end of the transaction, that 372 # of such operation now happens at the end of the transaction, that
373 # ui.quiet has not direct effect on the output. 373 # ui.quiet has not direct effect on the output.
375 # To preserve this intend use an inelegant hack, we fail to report 375 # To preserve this intend use an inelegant hack, we fail to report
376 # the change if `quiet` is set. We should probably move to 376 # the change if `quiet` is set. We should probably move to
377 # something better, but this is a good first step to allow the "end 377 # something better, but this is a good first step to allow the "end
378 # of transaction report" to pass tests. 378 # of transaction report" to pass tests.
379 if not repo.ui.quiet: 379 if not repo.ui.quiet:
380 tr.changes['changegroup-count-changesets'] += changesets 380 tr.changes[b'changegroup-count-changesets'] += changesets
381 tr.changes['changegroup-count-revisions'] += newrevs 381 tr.changes[b'changegroup-count-revisions'] += newrevs
382 tr.changes['changegroup-count-files'] += newfiles 382 tr.changes[b'changegroup-count-files'] += newfiles
383 383
384 deltaheads = 0 384 deltaheads = 0
385 if oldheads: 385 if oldheads:
386 heads = cl.heads() 386 heads = cl.heads()
387 deltaheads += len(heads) - len(oldheads) 387 deltaheads += len(heads) - len(oldheads)
389 if h not in oldheads and repo[h].closesbranch(): 389 if h not in oldheads and repo[h].closesbranch():
390 deltaheads -= 1 390 deltaheads -= 1
391 391
392 # see previous comment about checking ui.quiet 392 # see previous comment about checking ui.quiet
393 if not repo.ui.quiet: 393 if not repo.ui.quiet:
394 tr.changes['changegroup-count-heads'] += deltaheads 394 tr.changes[b'changegroup-count-heads'] += deltaheads
395 repo.invalidatevolatilesets() 395 repo.invalidatevolatilesets()
396 396
397 if changesets > 0: 397 if changesets > 0:
398 if 'node' not in tr.hookargs: 398 if b'node' not in tr.hookargs:
399 tr.hookargs['node'] = hex(cl.node(clstart)) 399 tr.hookargs[b'node'] = hex(cl.node(clstart))
400 tr.hookargs['node_last'] = hex(cl.node(clend - 1)) 400 tr.hookargs[b'node_last'] = hex(cl.node(clend - 1))
401 hookargs = dict(tr.hookargs) 401 hookargs = dict(tr.hookargs)
402 else: 402 else:
403 hookargs = dict(tr.hookargs) 403 hookargs = dict(tr.hookargs)
404 hookargs['node'] = hex(cl.node(clstart)) 404 hookargs[b'node'] = hex(cl.node(clstart))
405 hookargs['node_last'] = hex(cl.node(clend - 1)) 405 hookargs[b'node_last'] = hex(cl.node(clend - 1))
406 repo.hook( 406 repo.hook(
407 'pretxnchangegroup', 407 b'pretxnchangegroup',
408 throw=True, 408 throw=True,
409 **pycompat.strkwargs(hookargs) 409 **pycompat.strkwargs(hookargs)
410 ) 410 )
411 411
412 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)] 412 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
413 phaseall = None 413 phaseall = None
414 if srctype in ('push', 'serve'): 414 if srctype in (b'push', b'serve'):
415 # Old servers can not push the boundary themselves. 415 # Old servers can not push the boundary themselves.
416 # New servers won't push the boundary if changeset already 416 # New servers won't push the boundary if changeset already
417 # exists locally as secret 417 # exists locally as secret
418 # 418 #
419 # We should not use added here but the list of all change in 419 # We should not use added here but the list of all change in
440 # transaction closes. So it's possible for the changelog 440 # transaction closes. So it's possible for the changelog
441 # to have changed since we last saw it. 441 # to have changed since we last saw it.
442 if clstart >= len(repo): 442 if clstart >= len(repo):
443 return 443 return
444 444
445 repo.hook("changegroup", **pycompat.strkwargs(hookargs)) 445 repo.hook(b"changegroup", **pycompat.strkwargs(hookargs))
446 446
447 for n in added: 447 for n in added:
448 args = hookargs.copy() 448 args = hookargs.copy()
449 args['node'] = hex(n) 449 args[b'node'] = hex(n)
450 del args['node_last'] 450 del args[b'node_last']
451 repo.hook("incoming", **pycompat.strkwargs(args)) 451 repo.hook(b"incoming", **pycompat.strkwargs(args))
452 452
453 newheads = [h for h in repo.heads() if h not in oldheads] 453 newheads = [h for h in repo.heads() if h not in oldheads]
454 repo.ui.log( 454 repo.ui.log(
455 "incoming", 455 b"incoming",
456 "%d incoming changes - new heads: %s\n", 456 b"%d incoming changes - new heads: %s\n",
457 len(added), 457 len(added),
458 ', '.join([hex(c[:6]) for c in newheads]), 458 b', '.join([hex(c[:6]) for c in newheads]),
459 ) 459 )
460 460
461 tr.addpostclose( 461 tr.addpostclose(
462 'changegroup-runhooks-%020i' % clstart, 462 b'changegroup-runhooks-%020i' % clstart,
463 lambda tr: repo._afterlock(runhooks), 463 lambda tr: repo._afterlock(runhooks),
464 ) 464 )
465 finally: 465 finally:
466 repo.ui.flush() 466 repo.ui.flush()
467 # never return 0 here: 467 # never return 0 here:
492 remain the same. 492 remain the same.
493 """ 493 """
494 494
495 deltaheader = _CHANGEGROUPV2_DELTA_HEADER 495 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
496 deltaheadersize = deltaheader.size 496 deltaheadersize = deltaheader.size
497 version = '02' 497 version = b'02'
498 498
499 def _deltaheader(self, headertuple, prevnode): 499 def _deltaheader(self, headertuple, prevnode):
500 node, p1, p2, deltabase, cs = headertuple 500 node, p1, p2, deltabase, cs = headertuple
501 flags = 0 501 flags = 0
502 return node, p1, p2, deltabase, cs, flags 502 return node, p1, p2, deltabase, cs, flags
510 separating manifests and files. 510 separating manifests and files.
511 """ 511 """
512 512
513 deltaheader = _CHANGEGROUPV3_DELTA_HEADER 513 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
514 deltaheadersize = deltaheader.size 514 deltaheadersize = deltaheader.size
515 version = '03' 515 version = b'03'
516 _grouplistcount = 2 # One list of manifests and one list of files 516 _grouplistcount = 2 # One list of manifests and one list of files
517 517
518 def _deltaheader(self, headertuple, prevnode): 518 def _deltaheader(self, headertuple, prevnode):
519 node, p1, p2, deltabase, cs, flags = headertuple 519 node, p1, p2, deltabase, cs, flags = headertuple
520 return node, p1, p2, deltabase, cs, flags 520 return node, p1, p2, deltabase, cs, flags
521 521
522 def _unpackmanifests(self, repo, revmap, trp, prog): 522 def _unpackmanifests(self, repo, revmap, trp, prog):
523 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog) 523 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
524 for chunkdata in iter(self.filelogheader, {}): 524 for chunkdata in iter(self.filelogheader, {}):
525 # If we get here, there are directory manifests in the changegroup 525 # If we get here, there are directory manifests in the changegroup
526 d = chunkdata["filename"] 526 d = chunkdata[b"filename"]
527 repo.ui.debug("adding %s revisions\n" % d) 527 repo.ui.debug(b"adding %s revisions\n" % d)
528 deltas = self.deltaiter() 528 deltas = self.deltaiter()
529 if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp): 529 if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp):
530 raise error.Abort(_("received dir revlog group is empty")) 530 raise error.Abort(_(b"received dir revlog group is empty"))
531 531
532 532
533 class headerlessfixup(object): 533 class headerlessfixup(object):
534 def __init__(self, fh, h): 534 def __init__(self, fh, h):
535 self._h = h 535 self._h = h
661 if store.linkrev(i) == clrev: 661 if store.linkrev(i) == clrev:
662 return i 662 return i
663 # We failed to resolve a parent for this node, so 663 # We failed to resolve a parent for this node, so
664 # we crash the changegroup construction. 664 # we crash the changegroup construction.
665 raise error.Abort( 665 raise error.Abort(
666 'unable to resolve parent while packing %r %r' 666 b'unable to resolve parent while packing %r %r'
667 ' for changeset %r' % (store.indexfile, rev, clrev) 667 b' for changeset %r' % (store.indexfile, rev, clrev)
668 ) 668 )
669 669
670 return nullrev 670 return nullrev
671 671
672 if not linkparents or (store.parentrevs(rev) == (nullrev, nullrev)): 672 if not linkparents or (store.parentrevs(rev) == (nullrev, nullrev)):
708 cl = repo.changelog 708 cl = repo.changelog
709 709
710 if ischangelog: 710 if ischangelog:
711 # `hg log` shows changesets in storage order. To preserve order 711 # `hg log` shows changesets in storage order. To preserve order
712 # across clones, send out changesets in storage order. 712 # across clones, send out changesets in storage order.
713 nodesorder = 'storage' 713 nodesorder = b'storage'
714 elif ellipses: 714 elif ellipses:
715 nodes = _sortnodesellipsis(store, nodes, cl, lookup) 715 nodes = _sortnodesellipsis(store, nodes, cl, lookup)
716 nodesorder = 'nodes' 716 nodesorder = b'nodes'
717 else: 717 else:
718 nodesorder = None 718 nodesorder = None
719 719
720 # Perform ellipses filtering and revision massaging. We do this before 720 # Perform ellipses filtering and revision massaging. We do this before
721 # emitrevisions() because a) filtering out revisions creates less work 721 # emitrevisions() because a) filtering out revisions creates less work
775 # We expect the first pass to be fast, so we only engage the progress 775 # We expect the first pass to be fast, so we only engage the progress
776 # meter for constructing the revision deltas. 776 # meter for constructing the revision deltas.
777 progress = None 777 progress = None
778 if topic is not None: 778 if topic is not None:
779 progress = repo.ui.makeprogress( 779 progress = repo.ui.makeprogress(
780 topic, unit=_('chunks'), total=len(nodes) 780 topic, unit=_(b'chunks'), total=len(nodes)
781 ) 781 )
782 782
783 configtarget = repo.ui.config('devel', 'bundle.delta') 783 configtarget = repo.ui.config(b'devel', b'bundle.delta')
784 if configtarget not in ('', 'p1', 'full'): 784 if configtarget not in (b'', b'p1', b'full'):
785 msg = _("""config "devel.bundle.delta" as unknown value: %s""") 785 msg = _("""config "devel.bundle.delta" as unknown value: %s""")
786 repo.ui.warn(msg % configtarget) 786 repo.ui.warn(msg % configtarget)
787 787
788 deltamode = repository.CG_DELTAMODE_STD 788 deltamode = repository.CG_DELTAMODE_STD
789 if forcedeltaparentprev: 789 if forcedeltaparentprev:
790 deltamode = repository.CG_DELTAMODE_PREV 790 deltamode = repository.CG_DELTAMODE_PREV
791 elif configtarget == 'p1': 791 elif configtarget == b'p1':
792 deltamode = repository.CG_DELTAMODE_P1 792 deltamode = repository.CG_DELTAMODE_P1
793 elif configtarget == 'full': 793 elif configtarget == b'full':
794 deltamode = repository.CG_DELTAMODE_FULL 794 deltamode = repository.CG_DELTAMODE_FULL
795 795
796 revisions = store.emitrevisions( 796 revisions = store.emitrevisions(
797 nodes, 797 nodes,
798 nodesorder=nodesorder, 798 nodesorder=nodesorder,
908 """ 908 """
909 909
910 repo = self._repo 910 repo = self._repo
911 cl = repo.changelog 911 cl = repo.changelog
912 912
913 self._verbosenote(_('uncompressed size of bundle content:\n')) 913 self._verbosenote(_(b'uncompressed size of bundle content:\n'))
914 size = 0 914 size = 0
915 915
916 clstate, deltas = self._generatechangelog( 916 clstate, deltas = self._generatechangelog(
917 cl, clnodes, generate=changelog 917 cl, clnodes, generate=changelog
918 ) 918 )
923 923
924 close = closechunk() 924 close = closechunk()
925 size += len(close) 925 size += len(close)
926 yield closechunk() 926 yield closechunk()
927 927
928 self._verbosenote(_('%8.i (changelog)\n') % size) 928 self._verbosenote(_(b'%8.i (changelog)\n') % size)
929 929
930 clrevorder = clstate['clrevorder'] 930 clrevorder = clstate[b'clrevorder']
931 manifests = clstate['manifests'] 931 manifests = clstate[b'manifests']
932 changedfiles = clstate['changedfiles'] 932 changedfiles = clstate[b'changedfiles']
933 933
934 # We need to make sure that the linkrev in the changegroup refers to 934 # We need to make sure that the linkrev in the changegroup refers to
935 # the first changeset that introduced the manifest or file revision. 935 # the first changeset that introduced the manifest or file revision.
936 # The fastpath is usually safer than the slowpath, because the filelogs 936 # The fastpath is usually safer than the slowpath, because the filelogs
937 # are walked in revlog order. 937 # are walked in revlog order.
948 948
949 # Treemanifests don't work correctly with fastpathlinkrev 949 # Treemanifests don't work correctly with fastpathlinkrev
950 # either, because we don't discover which directory nodes to 950 # either, because we don't discover which directory nodes to
951 # send along with files. This could probably be fixed. 951 # send along with files. This could probably be fixed.
952 fastpathlinkrev = fastpathlinkrev and ( 952 fastpathlinkrev = fastpathlinkrev and (
953 'treemanifest' not in repo.requirements 953 b'treemanifest' not in repo.requirements
954 ) 954 )
955 955
956 fnodes = {} # needed file nodes 956 fnodes = {} # needed file nodes
957 957
958 size = 0 958 size = 0
961 clrevorder, 961 clrevorder,
962 fastpathlinkrev, 962 fastpathlinkrev,
963 manifests, 963 manifests,
964 fnodes, 964 fnodes,
965 source, 965 source,
966 clstate['clrevtomanifestrev'], 966 clstate[b'clrevtomanifestrev'],
967 ) 967 )
968 968
969 for tree, deltas in it: 969 for tree, deltas in it:
970 if tree: 970 if tree:
971 assert self.version == b'03' 971 assert self.version == b'03'
981 981
982 close = closechunk() 982 close = closechunk()
983 size += len(close) 983 size += len(close)
984 yield close 984 yield close
985 985
986 self._verbosenote(_('%8.i (manifests)\n') % size) 986 self._verbosenote(_(b'%8.i (manifests)\n') % size)
987 yield self._manifestsend 987 yield self._manifestsend
988 988
989 mfdicts = None 989 mfdicts = None
990 if self._ellipses and self._isshallow: 990 if self._ellipses and self._isshallow:
991 mfdicts = [ 991 mfdicts = [
1019 1019
1020 close = closechunk() 1020 close = closechunk()
1021 size += len(close) 1021 size += len(close)
1022 yield close 1022 yield close
1023 1023
1024 self._verbosenote(_('%8.i %s\n') % (size, path)) 1024 self._verbosenote(_(b'%8.i %s\n') % (size, path))
1025 1025
1026 yield closechunk() 1026 yield closechunk()
1027 1027
1028 if clnodes: 1028 if clnodes:
1029 repo.hook('outgoing', node=hex(clnodes[0]), source=source) 1029 repo.hook(b'outgoing', node=hex(clnodes[0]), source=source)
1030 1030
1031 def _generatechangelog(self, cl, nodes, generate=True): 1031 def _generatechangelog(self, cl, nodes, generate=True):
1032 """Generate data for changelog chunks. 1032 """Generate data for changelog chunks.
1033 1033
1034 Returns a 2-tuple of a dict containing state and an iterable of 1034 Returns a 2-tuple of a dict containing state and an iterable of
1043 mfl = self._repo.manifestlog 1043 mfl = self._repo.manifestlog
1044 changedfiles = set() 1044 changedfiles = set()
1045 clrevtomanifestrev = {} 1045 clrevtomanifestrev = {}
1046 1046
1047 state = { 1047 state = {
1048 'clrevorder': clrevorder, 1048 b'clrevorder': clrevorder,
1049 'manifests': manifests, 1049 b'manifests': manifests,
1050 'changedfiles': changedfiles, 1050 b'changedfiles': changedfiles,
1051 'clrevtomanifestrev': clrevtomanifestrev, 1051 b'clrevtomanifestrev': clrevtomanifestrev,
1052 } 1052 }
1053 1053
1054 if not (generate or self._ellipses): 1054 if not (generate or self._ellipses):
1055 # sort the nodes in storage order 1055 # sort the nodes in storage order
1056 nodes = sorted(nodes, key=cl.rev) 1056 nodes = sorted(nodes, key=cl.rev)
1114 nodes, 1114 nodes,
1115 True, 1115 True,
1116 lookupcl, 1116 lookupcl,
1117 self._forcedeltaparentprev, 1117 self._forcedeltaparentprev,
1118 ellipses=self._ellipses, 1118 ellipses=self._ellipses,
1119 topic=_('changesets'), 1119 topic=_(b'changesets'),
1120 clrevtolocalrev={}, 1120 clrevtolocalrev={},
1121 fullclnodes=self._fullclnodes, 1121 fullclnodes=self._fullclnodes,
1122 precomputedellipsis=self._precomputedellipsis, 1122 precomputedellipsis=self._precomputedellipsis,
1123 ) 1123 )
1124 1124
1139 `source` is unused here, but is used by extensions like remotefilelog to 1139 `source` is unused here, but is used by extensions like remotefilelog to
1140 change what is sent based in pulls vs pushes, etc. 1140 change what is sent based in pulls vs pushes, etc.
1141 """ 1141 """
1142 repo = self._repo 1142 repo = self._repo
1143 mfl = repo.manifestlog 1143 mfl = repo.manifestlog
1144 tmfnodes = {'': manifests} 1144 tmfnodes = {b'': manifests}
1145 1145
1146 # Callback for the manifest, used to collect linkrevs for filelog 1146 # Callback for the manifest, used to collect linkrevs for filelog
1147 # revisions. 1147 # revisions.
1148 # Returns the linkrev node (collected in lookupcl). 1148 # Returns the linkrev node (collected in lookupcl).
1149 def makelookupmflinknode(tree, nodes): 1149 def makelookupmflinknode(tree, nodes):
1168 treemanifests to send. 1168 treemanifests to send.
1169 """ 1169 """
1170 clnode = nodes[x] 1170 clnode = nodes[x]
1171 mdata = mfl.get(tree, x).readfast(shallow=True) 1171 mdata = mfl.get(tree, x).readfast(shallow=True)
1172 for p, n, fl in mdata.iterentries(): 1172 for p, n, fl in mdata.iterentries():
1173 if fl == 't': # subdirectory manifest 1173 if fl == b't': # subdirectory manifest
1174 subtree = tree + p + '/' 1174 subtree = tree + p + b'/'
1175 tmfclnodes = tmfnodes.setdefault(subtree, {}) 1175 tmfclnodes = tmfnodes.setdefault(subtree, {})
1176 tmfclnode = tmfclnodes.setdefault(n, clnode) 1176 tmfclnode = tmfclnodes.setdefault(n, clnode)
1177 if clrevorder[clnode] < clrevorder[tmfclnode]: 1177 if clrevorder[clnode] < clrevorder[tmfclnode]:
1178 tmfclnodes[n] = clnode 1178 tmfclnodes[n] = clnode
1179 else: 1179 else:
1218 prunednodes, 1218 prunednodes,
1219 False, 1219 False,
1220 lookupfn, 1220 lookupfn,
1221 self._forcedeltaparentprev, 1221 self._forcedeltaparentprev,
1222 ellipses=self._ellipses, 1222 ellipses=self._ellipses,
1223 topic=_('manifests'), 1223 topic=_(b'manifests'),
1224 clrevtolocalrev=clrevtolocalrev, 1224 clrevtolocalrev=clrevtolocalrev,
1225 fullclnodes=self._fullclnodes, 1225 fullclnodes=self._fullclnodes,
1226 precomputedellipsis=self._precomputedellipsis, 1226 precomputedellipsis=self._precomputedellipsis,
1227 ) 1227 )
1228 1228
1314 else: 1314 else:
1315 linknodes = normallinknodes 1315 linknodes = normallinknodes
1316 1316
1317 repo = self._repo 1317 repo = self._repo
1318 progress = repo.ui.makeprogress( 1318 progress = repo.ui.makeprogress(
1319 _('files'), unit=_('files'), total=len(changedfiles) 1319 _(b'files'), unit=_(b'files'), total=len(changedfiles)
1320 ) 1320 )
1321 for i, fname in enumerate(sorted(changedfiles)): 1321 for i, fname in enumerate(sorted(changedfiles)):
1322 filerevlog = repo.file(fname) 1322 filerevlog = repo.file(fname)
1323 if not filerevlog: 1323 if not filerevlog:
1324 raise error.Abort( 1324 raise error.Abort(
1325 _("empty or missing file data for %s") % fname 1325 _(b"empty or missing file data for %s") % fname
1326 ) 1326 )
1327 1327
1328 clrevtolocalrev.clear() 1328 clrevtolocalrev.clear()
1329 1329
1330 linkrevnodes = linknodes(filerevlog, fname) 1330 linkrevnodes = linknodes(filerevlog, fname)
1452 fullnodes=fullnodes, 1452 fullnodes=fullnodes,
1453 ) 1453 )
1454 1454
1455 1455
1456 _packermap = { 1456 _packermap = {
1457 '01': (_makecg1packer, cg1unpacker), 1457 b'01': (_makecg1packer, cg1unpacker),
1458 # cg2 adds support for exchanging generaldelta 1458 # cg2 adds support for exchanging generaldelta
1459 '02': (_makecg2packer, cg2unpacker), 1459 b'02': (_makecg2packer, cg2unpacker),
1460 # cg3 adds support for exchanging revlog flags and treemanifests 1460 # cg3 adds support for exchanging revlog flags and treemanifests
1461 '03': (_makecg3packer, cg3unpacker), 1461 b'03': (_makecg3packer, cg3unpacker),
1462 } 1462 }
1463 1463
1464 1464
1465 def allsupportedversions(repo): 1465 def allsupportedversions(repo):
1466 versions = set(_packermap.keys()) 1466 versions = set(_packermap.keys())
1467 needv03 = False 1467 needv03 = False
1468 if ( 1468 if (
1469 repo.ui.configbool('experimental', 'changegroup3') 1469 repo.ui.configbool(b'experimental', b'changegroup3')
1470 or repo.ui.configbool('experimental', 'treemanifest') 1470 or repo.ui.configbool(b'experimental', b'treemanifest')
1471 or 'treemanifest' in repo.requirements 1471 or b'treemanifest' in repo.requirements
1472 ): 1472 ):
1473 # we keep version 03 because we need to to exchange treemanifest data 1473 # we keep version 03 because we need to to exchange treemanifest data
1474 # 1474 #
1475 # we also keep vresion 01 and 02, because it is possible for repo to 1475 # we also keep vresion 01 and 02, because it is possible for repo to
1476 # contains both normal and tree manifest at the same time. so using 1476 # contains both normal and tree manifest at the same time. so using
1477 # older version to pull data is viable 1477 # older version to pull data is viable
1478 # 1478 #
1479 # (or even to push subset of history) 1479 # (or even to push subset of history)
1480 needv03 = True 1480 needv03 = True
1481 if not needv03: 1481 if not needv03:
1482 versions.discard('03') 1482 versions.discard(b'03')
1483 return versions 1483 return versions
1484 1484
1485 1485
1486 # Changegroup versions that can be applied to the repo 1486 # Changegroup versions that can be applied to the repo
1487 def supportedincomingversions(repo): 1487 def supportedincomingversions(repo):
1489 1489
1490 1490
1491 # Changegroup versions that can be created from the repo 1491 # Changegroup versions that can be created from the repo
1492 def supportedoutgoingversions(repo): 1492 def supportedoutgoingversions(repo):
1493 versions = allsupportedversions(repo) 1493 versions = allsupportedversions(repo)
1494 if 'treemanifest' in repo.requirements: 1494 if b'treemanifest' in repo.requirements:
1495 # Versions 01 and 02 support only flat manifests and it's just too 1495 # Versions 01 and 02 support only flat manifests and it's just too
1496 # expensive to convert between the flat manifest and tree manifest on 1496 # expensive to convert between the flat manifest and tree manifest on
1497 # the fly. Since tree manifests are hashed differently, all of history 1497 # the fly. Since tree manifests are hashed differently, all of history
1498 # would have to be converted. Instead, we simply don't even pretend to 1498 # would have to be converted. Instead, we simply don't even pretend to
1499 # support versions 01 and 02. 1499 # support versions 01 and 02.
1500 versions.discard('01') 1500 versions.discard(b'01')
1501 versions.discard('02') 1501 versions.discard(b'02')
1502 if repository.NARROW_REQUIREMENT in repo.requirements: 1502 if repository.NARROW_REQUIREMENT in repo.requirements:
1503 # Versions 01 and 02 don't support revlog flags, and we need to 1503 # Versions 01 and 02 don't support revlog flags, and we need to
1504 # support that for stripping and unbundling to work. 1504 # support that for stripping and unbundling to work.
1505 versions.discard('01') 1505 versions.discard(b'01')
1506 versions.discard('02') 1506 versions.discard(b'02')
1507 if LFS_REQUIREMENT in repo.requirements: 1507 if LFS_REQUIREMENT in repo.requirements:
1508 # Versions 01 and 02 don't support revlog flags, and we need to 1508 # Versions 01 and 02 don't support revlog flags, and we need to
1509 # mark LFS entries with REVIDX_EXTSTORED. 1509 # mark LFS entries with REVIDX_EXTSTORED.
1510 versions.discard('01') 1510 versions.discard(b'01')
1511 versions.discard('02') 1511 versions.discard(b'02')
1512 1512
1513 return versions 1513 return versions
1514 1514
1515 1515
1516 def localversion(repo): 1516 def localversion(repo):
1522 def safeversion(repo): 1522 def safeversion(repo):
1523 # Finds the smallest version that it's safe to assume clients of the repo 1523 # Finds the smallest version that it's safe to assume clients of the repo
1524 # will support. For example, all hg versions that support generaldelta also 1524 # will support. For example, all hg versions that support generaldelta also
1525 # support changegroup 02. 1525 # support changegroup 02.
1526 versions = supportedoutgoingversions(repo) 1526 versions = supportedoutgoingversions(repo)
1527 if 'generaldelta' in repo.requirements: 1527 if b'generaldelta' in repo.requirements:
1528 versions.discard('01') 1528 versions.discard(b'01')
1529 assert versions 1529 assert versions
1530 return min(versions) 1530 return min(versions)
1531 1531
1532 1532
1533 def getbundler( 1533 def getbundler(
1546 if matcher is None: 1546 if matcher is None:
1547 matcher = matchmod.always() 1547 matcher = matchmod.always()
1548 if oldmatcher is None: 1548 if oldmatcher is None:
1549 oldmatcher = matchmod.never() 1549 oldmatcher = matchmod.never()
1550 1550
1551 if version == '01' and not matcher.always(): 1551 if version == b'01' and not matcher.always():
1552 raise error.ProgrammingError( 1552 raise error.ProgrammingError(
1553 'version 01 changegroups do not support ' 'sparse file matchers' 1553 b'version 01 changegroups do not support ' b'sparse file matchers'
1554 ) 1554 )
1555 1555
1556 if ellipses and version in (b'01', b'02'): 1556 if ellipses and version in (b'01', b'02'):
1557 raise error.Abort( 1557 raise error.Abort(
1558 _( 1558 _(
1559 'ellipsis nodes require at least cg3 on client and server, ' 1559 b'ellipsis nodes require at least cg3 on client and server, '
1560 'but negotiated version %s' 1560 b'but negotiated version %s'
1561 ) 1561 )
1562 % version 1562 % version
1563 ) 1563 )
1564 1564
1565 # Requested files could include files not in the local store. So 1565 # Requested files could include files not in the local store. So
1582 def getunbundler(version, fh, alg, extras=None): 1582 def getunbundler(version, fh, alg, extras=None):
1583 return _packermap[version][1](fh, alg, extras=extras) 1583 return _packermap[version][1](fh, alg, extras=extras)
1584 1584
1585 1585
1586 def _changegroupinfo(repo, nodes, source): 1586 def _changegroupinfo(repo, nodes, source):
1587 if repo.ui.verbose or source == 'bundle': 1587 if repo.ui.verbose or source == b'bundle':
1588 repo.ui.status(_("%d changesets found\n") % len(nodes)) 1588 repo.ui.status(_(b"%d changesets found\n") % len(nodes))
1589 if repo.ui.debugflag: 1589 if repo.ui.debugflag:
1590 repo.ui.debug("list of changesets:\n") 1590 repo.ui.debug(b"list of changesets:\n")
1591 for node in nodes: 1591 for node in nodes:
1592 repo.ui.debug("%s\n" % hex(node)) 1592 repo.ui.debug(b"%s\n" % hex(node))
1593 1593
1594 1594
1595 def makechangegroup( 1595 def makechangegroup(
1596 repo, outgoing, version, source, fastpath=False, bundlecaps=None 1596 repo, outgoing, version, source, fastpath=False, bundlecaps=None
1597 ): 1597 ):
1605 ) 1605 )
1606 return getunbundler( 1606 return getunbundler(
1607 version, 1607 version,
1608 util.chunkbuffer(cgstream), 1608 util.chunkbuffer(cgstream),
1609 None, 1609 None,
1610 {'clcount': len(outgoing.missing)}, 1610 {b'clcount': len(outgoing.missing)},
1611 ) 1611 )
1612 1612
1613 1613
1614 def makestream( 1614 def makestream(
1615 repo, 1615 repo,
1632 heads.sort() 1632 heads.sort()
1633 fastpathlinkrev = fastpath or ( 1633 fastpathlinkrev = fastpath or (
1634 repo.filtername is None and heads == sorted(repo.heads()) 1634 repo.filtername is None and heads == sorted(repo.heads())
1635 ) 1635 )
1636 1636
1637 repo.hook('preoutgoing', throw=True, source=source) 1637 repo.hook(b'preoutgoing', throw=True, source=source)
1638 _changegroupinfo(repo, csets, source) 1638 _changegroupinfo(repo, csets, source)
1639 return bundler.generate(commonrevs, csets, fastpathlinkrev, source) 1639 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1640 1640
1641 1641
1642 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles): 1642 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1643 revisions = 0 1643 revisions = 0
1644 files = 0 1644 files = 0
1645 progress = repo.ui.makeprogress( 1645 progress = repo.ui.makeprogress(
1646 _('files'), unit=_('files'), total=expectedfiles 1646 _(b'files'), unit=_(b'files'), total=expectedfiles
1647 ) 1647 )
1648 for chunkdata in iter(source.filelogheader, {}): 1648 for chunkdata in iter(source.filelogheader, {}):
1649 files += 1 1649 files += 1
1650 f = chunkdata["filename"] 1650 f = chunkdata[b"filename"]
1651 repo.ui.debug("adding %s revisions\n" % f) 1651 repo.ui.debug(b"adding %s revisions\n" % f)
1652 progress.increment() 1652 progress.increment()
1653 fl = repo.file(f) 1653 fl = repo.file(f)
1654 o = len(fl) 1654 o = len(fl)
1655 try: 1655 try:
1656 deltas = source.deltaiter() 1656 deltas = source.deltaiter()
1657 if not fl.addgroup(deltas, revmap, trp): 1657 if not fl.addgroup(deltas, revmap, trp):
1658 raise error.Abort(_("received file revlog group is empty")) 1658 raise error.Abort(_(b"received file revlog group is empty"))
1659 except error.CensoredBaseError as e: 1659 except error.CensoredBaseError as e:
1660 raise error.Abort(_("received delta base is censored: %s") % e) 1660 raise error.Abort(_(b"received delta base is censored: %s") % e)
1661 revisions += len(fl) - o 1661 revisions += len(fl) - o
1662 if f in needfiles: 1662 if f in needfiles:
1663 needs = needfiles[f] 1663 needs = needfiles[f]
1664 for new in pycompat.xrange(o, len(fl)): 1664 for new in pycompat.xrange(o, len(fl)):
1665 n = fl.node(new) 1665 n = fl.node(new)
1666 if n in needs: 1666 if n in needs:
1667 needs.remove(n) 1667 needs.remove(n)
1668 else: 1668 else:
1669 raise error.Abort(_("received spurious file revlog entry")) 1669 raise error.Abort(_(b"received spurious file revlog entry"))
1670 if not needs: 1670 if not needs:
1671 del needfiles[f] 1671 del needfiles[f]
1672 progress.complete() 1672 progress.complete()
1673 1673
1674 for f, needs in needfiles.iteritems(): 1674 for f, needs in needfiles.iteritems():
1676 for n in needs: 1676 for n in needs:
1677 try: 1677 try:
1678 fl.rev(n) 1678 fl.rev(n)
1679 except error.LookupError: 1679 except error.LookupError:
1680 raise error.Abort( 1680 raise error.Abort(
1681 _('missing file data for %s:%s - run hg verify') 1681 _(b'missing file data for %s:%s - run hg verify')
1682 % (f, hex(n)) 1682 % (f, hex(n))
1683 ) 1683 )
1684 1684
1685 return revisions, files 1685 return revisions, files