Mercurial > hg
diff hgext/lfs/blobstore.py @ 43076:2372284d9457
formatting: blacken the codebase
This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.
Done with:
hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S
# skip-blame mass-reformatting only
# no-check-commit reformats foo_bar functions
Differential Revision: https://phab.mercurial-scm.org/D6971
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:45:02 -0400 |
parents | 698667eb7523 |
children | 687b865b95ad |
line wrap: on
line diff
--- a/hgext/lfs/blobstore.py Sat Oct 05 10:29:34 2019 -0400 +++ b/hgext/lfs/blobstore.py Sun Oct 06 09:45:02 2019 -0400 @@ -29,15 +29,14 @@ worker, ) -from mercurial.utils import ( - stringutil, -) +from mercurial.utils import stringutil from ..largefiles import lfutil # 64 bytes for SHA256 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') + class lfsvfs(vfsmod.vfs): def join(self, path): """split the path at first two characters, like: XX/XXXXX...""" @@ -56,18 +55,20 @@ prefixlen = len(pathutil.normasprefix(root)) oids = [] - for dirpath, dirs, files in os.walk(self.reljoin(self.base, path - or b''), - onerror=onerror): + for dirpath, dirs, files in os.walk( + self.reljoin(self.base, path or b''), onerror=onerror + ): dirpath = dirpath[prefixlen:] # Silently skip unexpected files and directories if len(dirpath) == 2: - oids.extend([dirpath + f for f in files - if _lfsre.match(dirpath + f)]) + oids.extend( + [dirpath + f for f in files if _lfsre.match(dirpath + f)] + ) yield ('', [], oids) + class nullvfs(lfsvfs): def __init__(self): pass @@ -80,8 +81,10 @@ # self.vfs. Raise the same error as a normal vfs when asked to read a # file that doesn't exist. The only difference is the full file path # isn't available in the error. - raise IOError(errno.ENOENT, - pycompat.sysstr(b'%s: No such file or directory' % oid)) + raise IOError( + errno.ENOENT, + pycompat.sysstr(b'%s: No such file or directory' % oid), + ) def walk(self, path=None, onerror=None): return (b'', [], []) @@ -89,6 +92,7 @@ def write(self, oid, data): pass + class filewithprogress(object): """a file-like object that supports __len__ and read. @@ -97,7 +101,7 @@ def __init__(self, fp, callback): self._fp = fp - self._callback = callback # func(readsize) + self._callback = callback # func(readsize) fp.seek(0, os.SEEK_END) self._len = fp.tell() fp.seek(0) @@ -117,6 +121,7 @@ self._fp = None return data + class local(object): """Local blobstore for large file contents. @@ -161,8 +166,9 @@ realoid = node.hex(sha256.digest()) if realoid != oid: - raise LfsCorruptionError(_(b'corrupt remote lfs object: %s') - % oid) + raise LfsCorruptionError( + _(b'corrupt remote lfs object: %s') % oid + ) self._linktousercache(oid) @@ -186,16 +192,16 @@ blob, but that doesn't happen when the server tells the client that it already has the blob. """ - if (not isinstance(self.cachevfs, nullvfs) - and not self.vfs.exists(oid)): + if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid): self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) def _linktousercache(self, oid): # XXX: should we verify the content of the cache, and hardlink back to # the local store on success, but truncate, write and link on failure? - if (not self.cachevfs.exists(oid) - and not isinstance(self.cachevfs, nullvfs)): + if not self.cachevfs.exists(oid) and not isinstance( + self.cachevfs, nullvfs + ): self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) @@ -240,6 +246,7 @@ False otherwise.""" return self.cachevfs.exists(oid) or self.vfs.exists(oid) + def _urlerrorreason(urlerror): '''Create a friendly message for the given URLError to be used in an LfsRemoteError message. @@ -250,7 +257,7 @@ inst = urlerror.reason if util.safehasattr(inst, 'reason'): - try: # usually it is in the form (errno, strerror) + try: # usually it is in the form (errno, strerror) reason = inst.reason.args[1] except (AttributeError, IndexError): # it might be anything, for example a string @@ -264,6 +271,7 @@ else: return stringutil.forcebytestr(urlerror) + class lfsauthhandler(util.urlreq.basehandler): handler_order = 480 # Before HTTPDigestAuthHandler (== 490) @@ -277,13 +285,17 @@ if scheme.lower() != r'basic': msg = _(b'the server must support Basic Authentication') - raise util.urlerr.httperror(req.get_full_url(), code, - encoding.strfromlocal(msg), headers, - fp) + raise util.urlerr.httperror( + req.get_full_url(), + code, + encoding.strfromlocal(msg), + headers, + fp, + ) return None + class _gitlfsremote(object): - def __init__(self, repo, url): ui = repo.ui self.ui = ui @@ -310,12 +322,15 @@ Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md """ - objects = [{r'oid': pycompat.strurl(p.oid()), - r'size': p.size()} for p in pointers] - requestdata = pycompat.bytesurl(json.dumps({ - r'objects': objects, - r'operation': pycompat.strurl(action), - })) + objects = [ + {r'oid': pycompat.strurl(p.oid()), r'size': p.size()} + for p in pointers + ] + requestdata = pycompat.bytesurl( + json.dumps( + {r'objects': objects, r'operation': pycompat.strurl(action),} + ) + ) url = b'%s/objects/batch' % self.baseurl batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json') @@ -325,41 +340,56 @@ rawjson = rsp.read() except util.urlerr.httperror as ex: hints = { - 400: _(b'check that lfs serving is enabled on %s and "%s" is ' - b'supported') % (self.baseurl, action), + 400: _( + b'check that lfs serving is enabled on %s and "%s" is ' + b'supported' + ) + % (self.baseurl, action), 404: _(b'the "lfs.url" config may be used to override %s') - % self.baseurl, + % self.baseurl, } hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) raise LfsRemoteError( _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), - hint=hint) + hint=hint, + ) except util.urlerr.urlerror as ex: - hint = (_(b'the "lfs.url" config may be used to override %s') - % self.baseurl) - raise LfsRemoteError(_(b'LFS error: %s') % _urlerrorreason(ex), - hint=hint) + hint = ( + _(b'the "lfs.url" config may be used to override %s') + % self.baseurl + ) + raise LfsRemoteError( + _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint + ) try: response = json.loads(rawjson) except ValueError: - raise LfsRemoteError(_(b'LFS server returns invalid JSON: %s') - % rawjson.encode("utf-8")) + raise LfsRemoteError( + _(b'LFS server returns invalid JSON: %s') + % rawjson.encode("utf-8") + ) if self.ui.debugflag: self.ui.debug(b'Status: %d\n' % rsp.status) # lfs-test-server and hg serve return headers in different order headers = pycompat.bytestr(rsp.info()).strip() - self.ui.debug(b'%s\n' - % b'\n'.join(sorted(headers.splitlines()))) + self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) if r'objects' in response: - response[r'objects'] = sorted(response[r'objects'], - key=lambda p: p[r'oid']) - self.ui.debug(b'%s\n' - % pycompat.bytesurl( - json.dumps(response, indent=2, - separators=(r'', r': '), - sort_keys=True))) + response[r'objects'] = sorted( + response[r'objects'], key=lambda p: p[r'oid'] + ) + self.ui.debug( + b'%s\n' + % pycompat.bytesurl( + json.dumps( + response, + indent=2, + separators=(r'', r': '), + sort_keys=True, + ) + ) + ) def encodestr(x): if isinstance(x, pycompat.unicode): @@ -378,8 +408,9 @@ # but just removes "download" from "actions". Treat that case # as the same as 404 error. if b'error' not in response: - if (action == b'download' - and action not in response.get(b'actions', [])): + if action == b'download' and action not in response.get( + b'actions', [] + ): code = 404 else: continue @@ -399,12 +430,14 @@ 500: b'Internal server error', } msg = errors.get(code, b'status code %d' % code) - raise LfsRemoteError(_(b'LFS server error for "%s": %s') - % (filename, msg)) + raise LfsRemoteError( + _(b'LFS server error for "%s": %s') % (filename, msg) + ) else: raise LfsRemoteError( _(b'LFS server error. Unsolicited response for oid %s') - % response[b'oid']) + % response[b'oid'] + ) def _extractobjects(self, response, pointers, action): """extract objects from response of the batch API @@ -419,8 +452,9 @@ # Filter objects with given action. Practically, this skips uploading # objects which exist in the server. - filteredobjects = [o for o in objects - if action in o.get(b'actions', [])] + filteredobjects = [ + o for o in objects if action in o.get(b'actions', []) + ] return filteredobjects @@ -442,8 +476,10 @@ if action == b'upload': # If uploading blobs, read data from local blobstore. if not localstore.verify(oid): - raise error.Abort(_(b'detected corrupt lfs object: %s') % oid, - hint=_(b'run hg verify')) + raise error.Abort( + _(b'detected corrupt lfs object: %s') % oid, + hint=_(b'run hg verify'), + ) request.data = filewithprogress(localstore.open(oid), None) request.get_method = lambda: r'PUT' request.add_header(r'Content-Type', r'application/octet-stream') @@ -461,8 +497,7 @@ # lfs-test-server and hg serve return headers in different # order headers = pycompat.bytestr(req.info()).strip() - ui.debug(b'%s\n' - % b'\n'.join(sorted(headers.splitlines()))) + ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) if action == b'download': # If downloading blobs, store downloaded data to local @@ -478,14 +513,20 @@ ui.debug(b'lfs %s response: %s' % (action, response)) except util.urlerr.httperror as ex: if self.ui.debugflag: - self.ui.debug(b'%s: %s\n' % (oid, ex.read())) # XXX: also bytes? - raise LfsRemoteError(_(b'LFS HTTP error: %s (oid=%s, action=%s)') - % (stringutil.forcebytestr(ex), oid, action)) + self.ui.debug( + b'%s: %s\n' % (oid, ex.read()) + ) # XXX: also bytes? + raise LfsRemoteError( + _(b'LFS HTTP error: %s (oid=%s, action=%s)') + % (stringutil.forcebytestr(ex), oid, action) + ) except util.urlerr.urlerror as ex: - hint = (_(b'attempted connection to %s') - % pycompat.bytesurl(util.urllibcompat.getfullurl(request))) - raise LfsRemoteError(_(b'LFS error: %s') % _urlerrorreason(ex), - hint=hint) + hint = _(b'attempted connection to %s') % pycompat.bytesurl( + util.urllibcompat.getfullurl(request) + ) + raise LfsRemoteError( + _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint + ) def _batch(self, pointers, localstore, action): if action not in [b'upload', b'download']: @@ -497,11 +538,15 @@ sizes = {} for obj in objects: sizes[obj.get(b'oid')] = obj.get(b'size', 0) - topic = {b'upload': _(b'lfs uploading'), - b'download': _(b'lfs downloading')}[action] + topic = { + b'upload': _(b'lfs uploading'), + b'download': _(b'lfs downloading'), + }[action] if len(objects) > 1: - self.ui.note(_(b'lfs: need to transfer %d objects (%s)\n') - % (len(objects), util.bytecount(total))) + self.ui.note( + _(b'lfs: need to transfer %d objects (%s)\n') + % (len(objects), util.bytecount(total)) + ) def transfer(chunk): for obj in chunk: @@ -511,8 +556,9 @@ msg = _(b'lfs: downloading %s (%s)\n') elif action == b'upload': msg = _(b'lfs: uploading %s (%s)\n') - self.ui.note(msg % (obj.get(b'oid'), - util.bytecount(objsize))) + self.ui.note( + msg % (obj.get(b'oid'), util.bytecount(objsize)) + ) retry = self.retry while True: try: @@ -523,15 +569,21 @@ if retry > 0: self.ui.note( _(b'lfs: failed: %r (remaining retry %d)\n') - % (stringutil.forcebytestr(ex), retry)) + % (stringutil.forcebytestr(ex), retry) + ) retry -= 1 continue raise # Until https multiplexing gets sorted out if self.ui.configbool(b'experimental', b'lfs.worker-enable'): - oids = worker.worker(self.ui, 0.1, transfer, (), - sorted(objects, key=lambda o: o.get(b'oid'))) + oids = worker.worker( + self.ui, + 0.1, + transfer, + (), + sorted(objects, key=lambda o: o.get(b'oid')), + ) else: oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) @@ -547,11 +599,15 @@ if blobs > 0: if action == b'upload': - self.ui.status(_(b'lfs: uploaded %d files (%s)\n') - % (blobs, util.bytecount(processed))) + self.ui.status( + _(b'lfs: uploaded %d files (%s)\n') + % (blobs, util.bytecount(processed)) + ) elif action == b'download': - self.ui.status(_(b'lfs: downloaded %d files (%s)\n') - % (blobs, util.bytecount(processed))) + self.ui.status( + _(b'lfs: downloaded %d files (%s)\n') + % (blobs, util.bytecount(processed)) + ) def __del__(self): # copied from mercurial/httppeer.py @@ -559,7 +615,8 @@ if urlopener: for h in urlopener.handlers: h.close() - getattr(h, "close_all", lambda : None)() + getattr(h, "close_all", lambda: None)() + class _dummyremote(object): """Dummy store storing blobs to temp directory.""" @@ -579,6 +636,7 @@ with self.vfs(p.oid(), b'rb') as fp: tostore.download(p.oid(), fp) + class _nullremote(object): """Null store storing blobs to /dev/null.""" @@ -591,6 +649,7 @@ def readbatch(self, pointers, tostore): pass + class _promptremote(object): """Prompt user to set lfs.url when accessed.""" @@ -606,6 +665,7 @@ def _prompt(self): raise error.Abort(_(b'lfs.url needs to be configured')) + _storemap = { b'https': _gitlfsremote, b'http': _gitlfsremote, @@ -614,6 +674,7 @@ None: _promptremote, } + def _deduplicate(pointers): """Remove any duplicate oids that exist in the list""" reduced = util.sortdict() @@ -621,11 +682,15 @@ reduced[p.oid()] = p return reduced.values() + def _verify(oid, content): realoid = node.hex(hashlib.sha256(content).digest()) if realoid != oid: - raise LfsCorruptionError(_(b'detected corrupt lfs object: %s') % oid, - hint=_(b'run hg verify')) + raise LfsCorruptionError( + _(b'detected corrupt lfs object: %s') % oid, + hint=_(b'run hg verify'), + ) + def remote(repo, remote=None): """remotestore factory. return a store in _storemap depending on config @@ -669,9 +734,11 @@ raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) return _storemap[scheme](repo, url) + class LfsRemoteError(error.StorageError): pass + class LfsCorruptionError(error.Abort): """Raised when a corrupt blob is detected, aborting an operation