Mercurial > hg
comparison hgext/largefiles/lfutil.py @ 43077:687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Done with
python3.7 contrib/byteify-strings.py -i $(hg files 'set:mercurial/**.py - mercurial/thirdparty/** + hgext/**.py - hgext/fsmonitor/pywatchman/** - mercurial/__init__.py')
black -l 80 -t py33 -S $(hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**" - hgext/fsmonitor/pywatchman/**')
# skip-blame mass-reformatting only
Differential Revision: https://phab.mercurial-scm.org/D6972
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:48:39 -0400 |
parents | 2372284d9457 |
children | eef9a2d67051 |
comparison
equal
deleted
inserted
replaced
43076:2372284d9457 | 43077:687b865b95ad |
---|---|
29 sparse, | 29 sparse, |
30 util, | 30 util, |
31 vfs as vfsmod, | 31 vfs as vfsmod, |
32 ) | 32 ) |
33 | 33 |
34 shortname = '.hglf' | 34 shortname = b'.hglf' |
35 shortnameslash = shortname + '/' | 35 shortnameslash = shortname + b'/' |
36 longname = 'largefiles' | 36 longname = b'largefiles' |
37 | 37 |
38 # -- Private worker functions ------------------------------------------ | 38 # -- Private worker functions ------------------------------------------ |
39 | 39 |
40 | 40 |
41 def getminsize(ui, assumelfiles, opt, default=10): | 41 def getminsize(ui, assumelfiles, opt, default=10): |
42 lfsize = opt | 42 lfsize = opt |
43 if not lfsize and assumelfiles: | 43 if not lfsize and assumelfiles: |
44 lfsize = ui.config(longname, 'minsize', default=default) | 44 lfsize = ui.config(longname, b'minsize', default=default) |
45 if lfsize: | 45 if lfsize: |
46 try: | 46 try: |
47 lfsize = float(lfsize) | 47 lfsize = float(lfsize) |
48 except ValueError: | 48 except ValueError: |
49 raise error.Abort( | 49 raise error.Abort( |
50 _('largefiles: size must be number (not %s)\n') % lfsize | 50 _(b'largefiles: size must be number (not %s)\n') % lfsize |
51 ) | 51 ) |
52 if lfsize is None: | 52 if lfsize is None: |
53 raise error.Abort(_('minimum size for largefiles must be specified')) | 53 raise error.Abort(_(b'minimum size for largefiles must be specified')) |
54 return lfsize | 54 return lfsize |
55 | 55 |
56 | 56 |
57 def link(src, dest): | 57 def link(src, dest): |
58 """Try to create hardlink - if that fails, efficiently make a copy.""" | 58 """Try to create hardlink - if that fails, efficiently make a copy.""" |
59 util.makedirs(os.path.dirname(dest)) | 59 util.makedirs(os.path.dirname(dest)) |
60 try: | 60 try: |
61 util.oslink(src, dest) | 61 util.oslink(src, dest) |
62 except OSError: | 62 except OSError: |
63 # if hardlinks fail, fallback on atomic copy | 63 # if hardlinks fail, fallback on atomic copy |
64 with open(src, 'rb') as srcf, util.atomictempfile(dest) as dstf: | 64 with open(src, b'rb') as srcf, util.atomictempfile(dest) as dstf: |
65 for chunk in util.filechunkiter(srcf): | 65 for chunk in util.filechunkiter(srcf): |
66 dstf.write(chunk) | 66 dstf.write(chunk) |
67 os.chmod(dest, os.stat(src).st_mode) | 67 os.chmod(dest, os.stat(src).st_mode) |
68 | 68 |
69 | 69 |
75 return os.path.join(_usercachedir(ui), hash) | 75 return os.path.join(_usercachedir(ui), hash) |
76 | 76 |
77 | 77 |
78 def _usercachedir(ui, name=longname): | 78 def _usercachedir(ui, name=longname): |
79 '''Return the location of the "global" largefiles cache.''' | 79 '''Return the location of the "global" largefiles cache.''' |
80 path = ui.configpath(name, 'usercache') | 80 path = ui.configpath(name, b'usercache') |
81 if path: | 81 if path: |
82 return path | 82 return path |
83 if pycompat.iswindows: | 83 if pycompat.iswindows: |
84 appdata = encoding.environ.get( | 84 appdata = encoding.environ.get( |
85 'LOCALAPPDATA', encoding.environ.get('APPDATA') | 85 b'LOCALAPPDATA', encoding.environ.get(b'APPDATA') |
86 ) | 86 ) |
87 if appdata: | 87 if appdata: |
88 return os.path.join(appdata, name) | 88 return os.path.join(appdata, name) |
89 elif pycompat.isdarwin: | 89 elif pycompat.isdarwin: |
90 home = encoding.environ.get('HOME') | 90 home = encoding.environ.get(b'HOME') |
91 if home: | 91 if home: |
92 return os.path.join(home, 'Library', 'Caches', name) | 92 return os.path.join(home, b'Library', b'Caches', name) |
93 elif pycompat.isposix: | 93 elif pycompat.isposix: |
94 path = encoding.environ.get('XDG_CACHE_HOME') | 94 path = encoding.environ.get(b'XDG_CACHE_HOME') |
95 if path: | 95 if path: |
96 return os.path.join(path, name) | 96 return os.path.join(path, name) |
97 home = encoding.environ.get('HOME') | 97 home = encoding.environ.get(b'HOME') |
98 if home: | 98 if home: |
99 return os.path.join(home, '.cache', name) | 99 return os.path.join(home, b'.cache', name) |
100 else: | 100 else: |
101 raise error.Abort(_('unknown operating system: %s\n') % pycompat.osname) | 101 raise error.Abort( |
102 raise error.Abort(_('unknown %s usercache location') % name) | 102 _(b'unknown operating system: %s\n') % pycompat.osname |
103 ) | |
104 raise error.Abort(_(b'unknown %s usercache location') % name) | |
103 | 105 |
104 | 106 |
105 def inusercache(ui, hash): | 107 def inusercache(ui, hash): |
106 path = usercachepath(ui, hash) | 108 path = usercachepath(ui, hash) |
107 return os.path.exists(path) | 109 return os.path.exists(path) |
111 '''Return store path of the largefile with the specified hash. | 113 '''Return store path of the largefile with the specified hash. |
112 As a side effect, the file might be linked from user cache. | 114 As a side effect, the file might be linked from user cache. |
113 Return None if the file can't be found locally.''' | 115 Return None if the file can't be found locally.''' |
114 path, exists = findstorepath(repo, hash) | 116 path, exists = findstorepath(repo, hash) |
115 if exists: | 117 if exists: |
116 repo.ui.note(_('found %s in store\n') % hash) | 118 repo.ui.note(_(b'found %s in store\n') % hash) |
117 return path | 119 return path |
118 elif inusercache(repo.ui, hash): | 120 elif inusercache(repo.ui, hash): |
119 repo.ui.note(_('found %s in system cache\n') % hash) | 121 repo.ui.note(_(b'found %s in system cache\n') % hash) |
120 path = storepath(repo, hash) | 122 path = storepath(repo, hash) |
121 link(usercachepath(repo.ui, hash), path) | 123 link(usercachepath(repo.ui, hash), path) |
122 return path | 124 return path |
123 return None | 125 return None |
124 | 126 |
172 ) | 174 ) |
173 | 175 |
174 # If the largefiles dirstate does not exist, populate and create | 176 # If the largefiles dirstate does not exist, populate and create |
175 # it. This ensures that we create it on the first meaningful | 177 # it. This ensures that we create it on the first meaningful |
176 # largefiles operation in a new clone. | 178 # largefiles operation in a new clone. |
177 if create and not vfs.exists(vfs.join(lfstoredir, 'dirstate')): | 179 if create and not vfs.exists(vfs.join(lfstoredir, b'dirstate')): |
178 matcher = getstandinmatcher(repo) | 180 matcher = getstandinmatcher(repo) |
179 standins = repo.dirstate.walk( | 181 standins = repo.dirstate.walk( |
180 matcher, subrepos=[], unknown=False, ignored=False | 182 matcher, subrepos=[], unknown=False, ignored=False |
181 ) | 183 ) |
182 | 184 |
188 lfdirstate.normallookup(lfile) | 190 lfdirstate.normallookup(lfile) |
189 return lfdirstate | 191 return lfdirstate |
190 | 192 |
191 | 193 |
192 def lfdirstatestatus(lfdirstate, repo): | 194 def lfdirstatestatus(lfdirstate, repo): |
193 pctx = repo['.'] | 195 pctx = repo[b'.'] |
194 match = matchmod.always() | 196 match = matchmod.always() |
195 unsure, s = lfdirstate.status( | 197 unsure, s = lfdirstate.status( |
196 match, subrepos=[], ignored=False, clean=False, unknown=False | 198 match, subrepos=[], ignored=False, clean=False, unknown=False |
197 ) | 199 ) |
198 modified, clean = s.modified, s.clean | 200 modified, clean = s.modified, s.clean |
218 | 220 |
219 # ignore unknown files in working directory | 221 # ignore unknown files in working directory |
220 return [ | 222 return [ |
221 splitstandin(f) | 223 splitstandin(f) |
222 for f in repo[rev].walk(matcher) | 224 for f in repo[rev].walk(matcher) |
223 if rev is not None or repo.dirstate[f] != '?' | 225 if rev is not None or repo.dirstate[f] != b'?' |
224 ] | 226 ] |
225 | 227 |
226 | 228 |
227 def instore(repo, hash, forcelocal=False): | 229 def instore(repo, hash, forcelocal=False): |
228 '''Return true if a largefile with the given hash exists in the store''' | 230 '''Return true if a largefile with the given hash exists in the store''' |
266 if path is None: | 268 if path is None: |
267 return False | 269 return False |
268 wvfs.makedirs(wvfs.dirname(wvfs.join(filename))) | 270 wvfs.makedirs(wvfs.dirname(wvfs.join(filename))) |
269 # The write may fail before the file is fully written, but we | 271 # The write may fail before the file is fully written, but we |
270 # don't use atomic writes in the working copy. | 272 # don't use atomic writes in the working copy. |
271 with open(path, 'rb') as srcfd, wvfs(filename, 'wb') as destfd: | 273 with open(path, b'rb') as srcfd, wvfs(filename, b'wb') as destfd: |
272 gothash = copyandhash(util.filechunkiter(srcfd), destfd) | 274 gothash = copyandhash(util.filechunkiter(srcfd), destfd) |
273 if gothash != hash: | 275 if gothash != hash: |
274 repo.ui.warn( | 276 repo.ui.warn( |
275 _('%s: data corruption in %s with hash %s\n') | 277 _(b'%s: data corruption in %s with hash %s\n') |
276 % (filename, path, gothash) | 278 % (filename, path, gothash) |
277 ) | 279 ) |
278 wvfs.unlink(filename) | 280 wvfs.unlink(filename) |
279 return False | 281 return False |
280 return True | 282 return True |
287 return | 289 return |
288 if wvfs.exists(file): | 290 if wvfs.exists(file): |
289 copytostoreabsolute(repo, wvfs.join(file), hash) | 291 copytostoreabsolute(repo, wvfs.join(file), hash) |
290 else: | 292 else: |
291 repo.ui.warn( | 293 repo.ui.warn( |
292 _("%s: largefile %s not available from local store\n") | 294 _(b"%s: largefile %s not available from local store\n") |
293 % (file, hash) | 295 % (file, hash) |
294 ) | 296 ) |
295 | 297 |
296 | 298 |
297 def copyalltostore(repo, node): | 299 def copyalltostore(repo, node): |
307 def copytostoreabsolute(repo, file, hash): | 309 def copytostoreabsolute(repo, file, hash): |
308 if inusercache(repo.ui, hash): | 310 if inusercache(repo.ui, hash): |
309 link(usercachepath(repo.ui, hash), storepath(repo, hash)) | 311 link(usercachepath(repo.ui, hash), storepath(repo, hash)) |
310 else: | 312 else: |
311 util.makedirs(os.path.dirname(storepath(repo, hash))) | 313 util.makedirs(os.path.dirname(storepath(repo, hash))) |
312 with open(file, 'rb') as srcf: | 314 with open(file, b'rb') as srcf: |
313 with util.atomictempfile( | 315 with util.atomictempfile( |
314 storepath(repo, hash), createmode=repo.store.createmode | 316 storepath(repo, hash), createmode=repo.store.createmode |
315 ) as dstf: | 317 ) as dstf: |
316 for chunk in util.filechunkiter(srcf): | 318 for chunk in util.filechunkiter(srcf): |
317 dstf.write(chunk) | 319 dstf.write(chunk) |
380 | 382 |
381 def splitstandin(filename): | 383 def splitstandin(filename): |
382 # Split on / because that's what dirstate always uses, even on Windows. | 384 # Split on / because that's what dirstate always uses, even on Windows. |
383 # Change local separator to / first just in case we are passed filenames | 385 # Change local separator to / first just in case we are passed filenames |
384 # from an external source (like the command line). | 386 # from an external source (like the command line). |
385 bits = util.pconvert(filename).split('/', 1) | 387 bits = util.pconvert(filename).split(b'/', 1) |
386 if len(bits) == 2 and bits[0] == shortname: | 388 if len(bits) == 2 and bits[0] == shortname: |
387 return bits[1] | 389 return bits[1] |
388 else: | 390 else: |
389 return None | 391 return None |
390 | 392 |
398 if repo.wvfs.exists(lfile): | 400 if repo.wvfs.exists(lfile): |
399 hash = hashfile(file) | 401 hash = hashfile(file) |
400 executable = getexecutable(file) | 402 executable = getexecutable(file) |
401 writestandin(repo, standin, hash, executable) | 403 writestandin(repo, standin, hash, executable) |
402 else: | 404 else: |
403 raise error.Abort(_('%s: file not found!') % lfile) | 405 raise error.Abort(_(b'%s: file not found!') % lfile) |
404 | 406 |
405 | 407 |
406 def readasstandin(fctx): | 408 def readasstandin(fctx): |
407 '''read hex hash from given filectx of standin file | 409 '''read hex hash from given filectx of standin file |
408 | 410 |
410 return fctx.data().strip() | 412 return fctx.data().strip() |
411 | 413 |
412 | 414 |
413 def writestandin(repo, standin, hash, executable): | 415 def writestandin(repo, standin, hash, executable): |
414 '''write hash to <repo.root>/<standin>''' | 416 '''write hash to <repo.root>/<standin>''' |
415 repo.wwrite(standin, hash + '\n', executable and 'x' or '') | 417 repo.wwrite(standin, hash + b'\n', executable and b'x' or b'') |
416 | 418 |
417 | 419 |
418 def copyandhash(instream, outfile): | 420 def copyandhash(instream, outfile): |
419 '''Read bytes from instream (iterable) and write them to outfile, | 421 '''Read bytes from instream (iterable) and write them to outfile, |
420 computing the SHA-1 hash of the data along the way. Return the hash.''' | 422 computing the SHA-1 hash of the data along the way. Return the hash.''' |
421 hasher = hashlib.sha1('') | 423 hasher = hashlib.sha1(b'') |
422 for data in instream: | 424 for data in instream: |
423 hasher.update(data) | 425 hasher.update(data) |
424 outfile.write(data) | 426 outfile.write(data) |
425 return hex(hasher.digest()) | 427 return hex(hasher.digest()) |
426 | 428 |
427 | 429 |
428 def hashfile(file): | 430 def hashfile(file): |
429 if not os.path.exists(file): | 431 if not os.path.exists(file): |
430 return '' | 432 return b'' |
431 with open(file, 'rb') as fd: | 433 with open(file, b'rb') as fd: |
432 return hexsha1(fd) | 434 return hexsha1(fd) |
433 | 435 |
434 | 436 |
435 def getexecutable(filename): | 437 def getexecutable(filename): |
436 mode = os.stat(filename).st_mode | 438 mode = os.stat(filename).st_mode |
441 ) | 443 ) |
442 | 444 |
443 | 445 |
444 def urljoin(first, second, *arg): | 446 def urljoin(first, second, *arg): |
445 def join(left, right): | 447 def join(left, right): |
446 if not left.endswith('/'): | 448 if not left.endswith(b'/'): |
447 left += '/' | 449 left += b'/' |
448 if right.startswith('/'): | 450 if right.startswith(b'/'): |
449 right = right[1:] | 451 right = right[1:] |
450 return left + right | 452 return left + right |
451 | 453 |
452 url = join(first, second) | 454 url = join(first, second) |
453 for a in arg: | 455 for a in arg: |
463 h.update(chunk) | 465 h.update(chunk) |
464 return hex(h.digest()) | 466 return hex(h.digest()) |
465 | 467 |
466 | 468 |
467 def httpsendfile(ui, filename): | 469 def httpsendfile(ui, filename): |
468 return httpconnection.httpsendfile(ui, filename, 'rb') | 470 return httpconnection.httpsendfile(ui, filename, b'rb') |
469 | 471 |
470 | 472 |
471 def unixpath(path): | 473 def unixpath(path): |
472 '''Return a version of path normalized for use with the lfdirstate.''' | 474 '''Return a version of path normalized for use with the lfdirstate.''' |
473 return util.pconvert(os.path.normpath(path)) | 475 return util.pconvert(os.path.normpath(path)) |
474 | 476 |
475 | 477 |
476 def islfilesrepo(repo): | 478 def islfilesrepo(repo): |
477 '''Return true if the repo is a largefile repo.''' | 479 '''Return true if the repo is a largefile repo.''' |
478 if 'largefiles' in repo.requirements and any( | 480 if b'largefiles' in repo.requirements and any( |
479 shortnameslash in f[0] for f in repo.store.datafiles() | 481 shortnameslash in f[0] for f in repo.store.datafiles() |
480 ): | 482 ): |
481 return True | 483 return True |
482 | 484 |
483 return any(openlfdirstate(repo.ui, repo, False)) | 485 return any(openlfdirstate(repo.ui, repo, False)) |
508 lfstandin = standin(lfile) | 510 lfstandin = standin(lfile) |
509 if lfstandin in repo.dirstate: | 511 if lfstandin in repo.dirstate: |
510 stat = repo.dirstate._map[lfstandin] | 512 stat = repo.dirstate._map[lfstandin] |
511 state, mtime = stat[0], stat[3] | 513 state, mtime = stat[0], stat[3] |
512 else: | 514 else: |
513 state, mtime = '?', -1 | 515 state, mtime = b'?', -1 |
514 if state == 'n': | 516 if state == b'n': |
515 if normallookup or mtime < 0 or not repo.wvfs.exists(lfile): | 517 if normallookup or mtime < 0 or not repo.wvfs.exists(lfile): |
516 # state 'n' doesn't ensure 'clean' in this case | 518 # state 'n' doesn't ensure 'clean' in this case |
517 lfdirstate.normallookup(lfile) | 519 lfdirstate.normallookup(lfile) |
518 else: | 520 else: |
519 lfdirstate.normal(lfile) | 521 lfdirstate.normal(lfile) |
520 elif state == 'm': | 522 elif state == b'm': |
521 lfdirstate.normallookup(lfile) | 523 lfdirstate.normallookup(lfile) |
522 elif state == 'r': | 524 elif state == b'r': |
523 lfdirstate.remove(lfile) | 525 lfdirstate.remove(lfile) |
524 elif state == 'a': | 526 elif state == b'a': |
525 lfdirstate.add(lfile) | 527 lfdirstate.add(lfile) |
526 elif state == '?': | 528 elif state == b'?': |
527 lfdirstate.drop(lfile) | 529 lfdirstate.drop(lfile) |
528 | 530 |
529 | 531 |
530 def markcommitted(orig, ctx, node): | 532 def markcommitted(orig, ctx, node): |
531 repo = ctx.repo() | 533 repo = ctx.repo() |
567 | 569 |
568 | 570 |
569 def getlfilestoupload(repo, missing, addfunc): | 571 def getlfilestoupload(repo, missing, addfunc): |
570 makeprogress = repo.ui.makeprogress | 572 makeprogress = repo.ui.makeprogress |
571 with makeprogress( | 573 with makeprogress( |
572 _('finding outgoing largefiles'), | 574 _(b'finding outgoing largefiles'), |
573 unit=_('revisions'), | 575 unit=_(b'revisions'), |
574 total=len(missing), | 576 total=len(missing), |
575 ) as progress: | 577 ) as progress: |
576 for i, n in enumerate(missing): | 578 for i, n in enumerate(missing): |
577 progress.update(i) | 579 progress.update(i) |
578 parents = [p for p in repo[n].parents() if p != node.nullid] | 580 parents = [p for p in repo[n].parents() if p != node.nullid] |
663 # asked to commit them, so sooner or later we're going to | 665 # asked to commit them, so sooner or later we're going to |
664 # refresh the standins. Might as well leave them refreshed. | 666 # refresh the standins. Might as well leave them refreshed. |
665 lfdirstate = openlfdirstate(ui, repo) | 667 lfdirstate = openlfdirstate(ui, repo) |
666 for fstandin in standins: | 668 for fstandin in standins: |
667 lfile = splitstandin(fstandin) | 669 lfile = splitstandin(fstandin) |
668 if lfdirstate[lfile] != 'r': | 670 if lfdirstate[lfile] != b'r': |
669 updatestandin(repo, lfile, fstandin) | 671 updatestandin(repo, lfile, fstandin) |
670 | 672 |
671 # Cook up a new matcher that only matches regular files or | 673 # Cook up a new matcher that only matches regular files or |
672 # standins corresponding to the big files requested by the | 674 # standins corresponding to the big files requested by the |
673 # user. Have to modify _files to prevent commit() from | 675 # user. Have to modify _files to prevent commit() from |
687 # For largefiles, only one of the normal and standin should be | 689 # For largefiles, only one of the normal and standin should be |
688 # committed (except if one of them is a remove). In the case of a | 690 # committed (except if one of them is a remove). In the case of a |
689 # standin removal, drop the normal file if it is unknown to dirstate. | 691 # standin removal, drop the normal file if it is unknown to dirstate. |
690 # Thus, skip plain largefile names but keep the standin. | 692 # Thus, skip plain largefile names but keep the standin. |
691 if f in lfiles or fstandin in standins: | 693 if f in lfiles or fstandin in standins: |
692 if repo.dirstate[fstandin] != 'r': | 694 if repo.dirstate[fstandin] != b'r': |
693 if repo.dirstate[f] != 'r': | 695 if repo.dirstate[f] != b'r': |
694 continue | 696 continue |
695 elif repo.dirstate[f] == '?': | 697 elif repo.dirstate[f] == b'?': |
696 continue | 698 continue |
697 | 699 |
698 actualfiles.append(f) | 700 actualfiles.append(f) |
699 match._files = actualfiles | 701 match._files = actualfiles |
700 | 702 |
739 ``repo._lfstatuswriters`` as "default" writer function. | 741 ``repo._lfstatuswriters`` as "default" writer function. |
740 | 742 |
741 Otherwise, this returns the function to always write out (or | 743 Otherwise, this returns the function to always write out (or |
742 ignore if ``not forcibly``) status. | 744 ignore if ``not forcibly``) status. |
743 ''' | 745 ''' |
744 if forcibly is None and util.safehasattr(repo, '_largefilesenabled'): | 746 if forcibly is None and util.safehasattr(repo, b'_largefilesenabled'): |
745 return repo._lfstatuswriters[-1] | 747 return repo._lfstatuswriters[-1] |
746 else: | 748 else: |
747 if forcibly: | 749 if forcibly: |
748 return ui.status # forcibly WRITE OUT | 750 return ui.status # forcibly WRITE OUT |
749 else: | 751 else: |