Mercurial > hg
comparison hgext/lfs/blobstore.py @ 43076:2372284d9457
formatting: blacken the codebase
This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.
Done with:
hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S
# skip-blame mass-reformatting only
# no-check-commit reformats foo_bar functions
Differential Revision: https://phab.mercurial-scm.org/D6971
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:45:02 -0400 |
parents | 698667eb7523 |
children | 687b865b95ad |
comparison
equal
deleted
inserted
replaced
43075:57875cf423c9 | 43076:2372284d9457 |
---|---|
27 util, | 27 util, |
28 vfs as vfsmod, | 28 vfs as vfsmod, |
29 worker, | 29 worker, |
30 ) | 30 ) |
31 | 31 |
32 from mercurial.utils import ( | 32 from mercurial.utils import stringutil |
33 stringutil, | |
34 ) | |
35 | 33 |
36 from ..largefiles import lfutil | 34 from ..largefiles import lfutil |
37 | 35 |
38 # 64 bytes for SHA256 | 36 # 64 bytes for SHA256 |
39 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') | 37 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') |
38 | |
40 | 39 |
41 class lfsvfs(vfsmod.vfs): | 40 class lfsvfs(vfsmod.vfs): |
42 def join(self, path): | 41 def join(self, path): |
43 """split the path at first two characters, like: XX/XXXXX...""" | 42 """split the path at first two characters, like: XX/XXXXX...""" |
44 if not _lfsre.match(path): | 43 if not _lfsre.match(path): |
54 # when dirpath == root, dirpath[prefixlen:] becomes empty | 53 # when dirpath == root, dirpath[prefixlen:] becomes empty |
55 # because len(dirpath) < prefixlen. | 54 # because len(dirpath) < prefixlen. |
56 prefixlen = len(pathutil.normasprefix(root)) | 55 prefixlen = len(pathutil.normasprefix(root)) |
57 oids = [] | 56 oids = [] |
58 | 57 |
59 for dirpath, dirs, files in os.walk(self.reljoin(self.base, path | 58 for dirpath, dirs, files in os.walk( |
60 or b''), | 59 self.reljoin(self.base, path or b''), onerror=onerror |
61 onerror=onerror): | 60 ): |
62 dirpath = dirpath[prefixlen:] | 61 dirpath = dirpath[prefixlen:] |
63 | 62 |
64 # Silently skip unexpected files and directories | 63 # Silently skip unexpected files and directories |
65 if len(dirpath) == 2: | 64 if len(dirpath) == 2: |
66 oids.extend([dirpath + f for f in files | 65 oids.extend( |
67 if _lfsre.match(dirpath + f)]) | 66 [dirpath + f for f in files if _lfsre.match(dirpath + f)] |
67 ) | |
68 | 68 |
69 yield ('', [], oids) | 69 yield ('', [], oids) |
70 | |
70 | 71 |
71 class nullvfs(lfsvfs): | 72 class nullvfs(lfsvfs): |
72 def __init__(self): | 73 def __init__(self): |
73 pass | 74 pass |
74 | 75 |
78 def read(self, oid): | 79 def read(self, oid): |
79 # store.read() calls into here if the blob doesn't exist in its | 80 # store.read() calls into here if the blob doesn't exist in its |
80 # self.vfs. Raise the same error as a normal vfs when asked to read a | 81 # self.vfs. Raise the same error as a normal vfs when asked to read a |
81 # file that doesn't exist. The only difference is the full file path | 82 # file that doesn't exist. The only difference is the full file path |
82 # isn't available in the error. | 83 # isn't available in the error. |
83 raise IOError(errno.ENOENT, | 84 raise IOError( |
84 pycompat.sysstr(b'%s: No such file or directory' % oid)) | 85 errno.ENOENT, |
86 pycompat.sysstr(b'%s: No such file or directory' % oid), | |
87 ) | |
85 | 88 |
86 def walk(self, path=None, onerror=None): | 89 def walk(self, path=None, onerror=None): |
87 return (b'', [], []) | 90 return (b'', [], []) |
88 | 91 |
89 def write(self, oid, data): | 92 def write(self, oid, data): |
90 pass | 93 pass |
91 | 94 |
95 | |
92 class filewithprogress(object): | 96 class filewithprogress(object): |
93 """a file-like object that supports __len__ and read. | 97 """a file-like object that supports __len__ and read. |
94 | 98 |
95 Useful to provide progress information for how many bytes are read. | 99 Useful to provide progress information for how many bytes are read. |
96 """ | 100 """ |
97 | 101 |
98 def __init__(self, fp, callback): | 102 def __init__(self, fp, callback): |
99 self._fp = fp | 103 self._fp = fp |
100 self._callback = callback # func(readsize) | 104 self._callback = callback # func(readsize) |
101 fp.seek(0, os.SEEK_END) | 105 fp.seek(0, os.SEEK_END) |
102 self._len = fp.tell() | 106 self._len = fp.tell() |
103 fp.seek(0) | 107 fp.seek(0) |
104 | 108 |
105 def __len__(self): | 109 def __len__(self): |
115 else: | 119 else: |
116 self._fp.close() | 120 self._fp.close() |
117 self._fp = None | 121 self._fp = None |
118 return data | 122 return data |
119 | 123 |
124 | |
120 class local(object): | 125 class local(object): |
121 """Local blobstore for large file contents. | 126 """Local blobstore for large file contents. |
122 | 127 |
123 This blobstore is used both as a cache and as a staging area for large blobs | 128 This blobstore is used both as a cache and as a staging area for large blobs |
124 to be uploaded to the remote blobstore. | 129 to be uploaded to the remote blobstore. |
159 fp.write(chunk) | 164 fp.write(chunk) |
160 sha256.update(chunk) | 165 sha256.update(chunk) |
161 | 166 |
162 realoid = node.hex(sha256.digest()) | 167 realoid = node.hex(sha256.digest()) |
163 if realoid != oid: | 168 if realoid != oid: |
164 raise LfsCorruptionError(_(b'corrupt remote lfs object: %s') | 169 raise LfsCorruptionError( |
165 % oid) | 170 _(b'corrupt remote lfs object: %s') % oid |
171 ) | |
166 | 172 |
167 self._linktousercache(oid) | 173 self._linktousercache(oid) |
168 | 174 |
169 def write(self, oid, data): | 175 def write(self, oid, data): |
170 """Write blob to local blobstore. | 176 """Write blob to local blobstore. |
184 upload the blob, to ensure it is always available in this store. | 190 upload the blob, to ensure it is always available in this store. |
185 Normally this is done implicitly when the client reads or writes the | 191 Normally this is done implicitly when the client reads or writes the |
186 blob, but that doesn't happen when the server tells the client that it | 192 blob, but that doesn't happen when the server tells the client that it |
187 already has the blob. | 193 already has the blob. |
188 """ | 194 """ |
189 if (not isinstance(self.cachevfs, nullvfs) | 195 if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid): |
190 and not self.vfs.exists(oid)): | |
191 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) | 196 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
192 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) | 197 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
193 | 198 |
194 def _linktousercache(self, oid): | 199 def _linktousercache(self, oid): |
195 # XXX: should we verify the content of the cache, and hardlink back to | 200 # XXX: should we verify the content of the cache, and hardlink back to |
196 # the local store on success, but truncate, write and link on failure? | 201 # the local store on success, but truncate, write and link on failure? |
197 if (not self.cachevfs.exists(oid) | 202 if not self.cachevfs.exists(oid) and not isinstance( |
198 and not isinstance(self.cachevfs, nullvfs)): | 203 self.cachevfs, nullvfs |
204 ): | |
199 self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) | 205 self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) |
200 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) | 206 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
201 | 207 |
202 def read(self, oid, verify=True): | 208 def read(self, oid, verify=True): |
203 """Read blob from local blobstore.""" | 209 """Read blob from local blobstore.""" |
238 def has(self, oid): | 244 def has(self, oid): |
239 """Returns True if the local blobstore contains the requested blob, | 245 """Returns True if the local blobstore contains the requested blob, |
240 False otherwise.""" | 246 False otherwise.""" |
241 return self.cachevfs.exists(oid) or self.vfs.exists(oid) | 247 return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
242 | 248 |
249 | |
243 def _urlerrorreason(urlerror): | 250 def _urlerrorreason(urlerror): |
244 '''Create a friendly message for the given URLError to be used in an | 251 '''Create a friendly message for the given URLError to be used in an |
245 LfsRemoteError message. | 252 LfsRemoteError message. |
246 ''' | 253 ''' |
247 inst = urlerror | 254 inst = urlerror |
248 | 255 |
249 if isinstance(urlerror.reason, Exception): | 256 if isinstance(urlerror.reason, Exception): |
250 inst = urlerror.reason | 257 inst = urlerror.reason |
251 | 258 |
252 if util.safehasattr(inst, 'reason'): | 259 if util.safehasattr(inst, 'reason'): |
253 try: # usually it is in the form (errno, strerror) | 260 try: # usually it is in the form (errno, strerror) |
254 reason = inst.reason.args[1] | 261 reason = inst.reason.args[1] |
255 except (AttributeError, IndexError): | 262 except (AttributeError, IndexError): |
256 # it might be anything, for example a string | 263 # it might be anything, for example a string |
257 reason = inst.reason | 264 reason = inst.reason |
258 if isinstance(reason, pycompat.unicode): | 265 if isinstance(reason, pycompat.unicode): |
262 elif getattr(inst, "strerror", None): | 269 elif getattr(inst, "strerror", None): |
263 return encoding.strtolocal(inst.strerror) | 270 return encoding.strtolocal(inst.strerror) |
264 else: | 271 else: |
265 return stringutil.forcebytestr(urlerror) | 272 return stringutil.forcebytestr(urlerror) |
266 | 273 |
274 | |
267 class lfsauthhandler(util.urlreq.basehandler): | 275 class lfsauthhandler(util.urlreq.basehandler): |
268 handler_order = 480 # Before HTTPDigestAuthHandler (== 490) | 276 handler_order = 480 # Before HTTPDigestAuthHandler (== 490) |
269 | 277 |
270 def http_error_401(self, req, fp, code, msg, headers): | 278 def http_error_401(self, req, fp, code, msg, headers): |
271 """Enforces that any authentication performed is HTTP Basic | 279 """Enforces that any authentication performed is HTTP Basic |
275 if authreq: | 283 if authreq: |
276 scheme = authreq.split()[0] | 284 scheme = authreq.split()[0] |
277 | 285 |
278 if scheme.lower() != r'basic': | 286 if scheme.lower() != r'basic': |
279 msg = _(b'the server must support Basic Authentication') | 287 msg = _(b'the server must support Basic Authentication') |
280 raise util.urlerr.httperror(req.get_full_url(), code, | 288 raise util.urlerr.httperror( |
281 encoding.strfromlocal(msg), headers, | 289 req.get_full_url(), |
282 fp) | 290 code, |
291 encoding.strfromlocal(msg), | |
292 headers, | |
293 fp, | |
294 ) | |
283 return None | 295 return None |
284 | 296 |
297 | |
285 class _gitlfsremote(object): | 298 class _gitlfsremote(object): |
286 | |
287 def __init__(self, repo, url): | 299 def __init__(self, repo, url): |
288 ui = repo.ui | 300 ui = repo.ui |
289 self.ui = ui | 301 self.ui = ui |
290 baseurl, authinfo = url.authinfo() | 302 baseurl, authinfo = url.authinfo() |
291 self.baseurl = baseurl.rstrip(b'/') | 303 self.baseurl = baseurl.rstrip(b'/') |
308 """Get metadata about objects pointed by pointers for given action | 320 """Get metadata about objects pointed by pointers for given action |
309 | 321 |
310 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} | 322 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
311 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md | 323 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
312 """ | 324 """ |
313 objects = [{r'oid': pycompat.strurl(p.oid()), | 325 objects = [ |
314 r'size': p.size()} for p in pointers] | 326 {r'oid': pycompat.strurl(p.oid()), r'size': p.size()} |
315 requestdata = pycompat.bytesurl(json.dumps({ | 327 for p in pointers |
316 r'objects': objects, | 328 ] |
317 r'operation': pycompat.strurl(action), | 329 requestdata = pycompat.bytesurl( |
318 })) | 330 json.dumps( |
331 {r'objects': objects, r'operation': pycompat.strurl(action),} | |
332 ) | |
333 ) | |
319 url = b'%s/objects/batch' % self.baseurl | 334 url = b'%s/objects/batch' % self.baseurl |
320 batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) | 335 batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) |
321 batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json') | 336 batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json') |
322 batchreq.add_header(r'Content-Type', r'application/vnd.git-lfs+json') | 337 batchreq.add_header(r'Content-Type', r'application/vnd.git-lfs+json') |
323 try: | 338 try: |
324 with contextlib.closing(self.urlopener.open(batchreq)) as rsp: | 339 with contextlib.closing(self.urlopener.open(batchreq)) as rsp: |
325 rawjson = rsp.read() | 340 rawjson = rsp.read() |
326 except util.urlerr.httperror as ex: | 341 except util.urlerr.httperror as ex: |
327 hints = { | 342 hints = { |
328 400: _(b'check that lfs serving is enabled on %s and "%s" is ' | 343 400: _( |
329 b'supported') % (self.baseurl, action), | 344 b'check that lfs serving is enabled on %s and "%s" is ' |
345 b'supported' | |
346 ) | |
347 % (self.baseurl, action), | |
330 404: _(b'the "lfs.url" config may be used to override %s') | 348 404: _(b'the "lfs.url" config may be used to override %s') |
331 % self.baseurl, | 349 % self.baseurl, |
332 } | 350 } |
333 hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) | 351 hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) |
334 raise LfsRemoteError( | 352 raise LfsRemoteError( |
335 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), | 353 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), |
336 hint=hint) | 354 hint=hint, |
355 ) | |
337 except util.urlerr.urlerror as ex: | 356 except util.urlerr.urlerror as ex: |
338 hint = (_(b'the "lfs.url" config may be used to override %s') | 357 hint = ( |
339 % self.baseurl) | 358 _(b'the "lfs.url" config may be used to override %s') |
340 raise LfsRemoteError(_(b'LFS error: %s') % _urlerrorreason(ex), | 359 % self.baseurl |
341 hint=hint) | 360 ) |
361 raise LfsRemoteError( | |
362 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint | |
363 ) | |
342 try: | 364 try: |
343 response = json.loads(rawjson) | 365 response = json.loads(rawjson) |
344 except ValueError: | 366 except ValueError: |
345 raise LfsRemoteError(_(b'LFS server returns invalid JSON: %s') | 367 raise LfsRemoteError( |
346 % rawjson.encode("utf-8")) | 368 _(b'LFS server returns invalid JSON: %s') |
369 % rawjson.encode("utf-8") | |
370 ) | |
347 | 371 |
348 if self.ui.debugflag: | 372 if self.ui.debugflag: |
349 self.ui.debug(b'Status: %d\n' % rsp.status) | 373 self.ui.debug(b'Status: %d\n' % rsp.status) |
350 # lfs-test-server and hg serve return headers in different order | 374 # lfs-test-server and hg serve return headers in different order |
351 headers = pycompat.bytestr(rsp.info()).strip() | 375 headers = pycompat.bytestr(rsp.info()).strip() |
352 self.ui.debug(b'%s\n' | 376 self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
353 % b'\n'.join(sorted(headers.splitlines()))) | |
354 | 377 |
355 if r'objects' in response: | 378 if r'objects' in response: |
356 response[r'objects'] = sorted(response[r'objects'], | 379 response[r'objects'] = sorted( |
357 key=lambda p: p[r'oid']) | 380 response[r'objects'], key=lambda p: p[r'oid'] |
358 self.ui.debug(b'%s\n' | 381 ) |
359 % pycompat.bytesurl( | 382 self.ui.debug( |
360 json.dumps(response, indent=2, | 383 b'%s\n' |
361 separators=(r'', r': '), | 384 % pycompat.bytesurl( |
362 sort_keys=True))) | 385 json.dumps( |
386 response, | |
387 indent=2, | |
388 separators=(r'', r': '), | |
389 sort_keys=True, | |
390 ) | |
391 ) | |
392 ) | |
363 | 393 |
364 def encodestr(x): | 394 def encodestr(x): |
365 if isinstance(x, pycompat.unicode): | 395 if isinstance(x, pycompat.unicode): |
366 return x.encode(u'utf-8') | 396 return x.encode(u'utf-8') |
367 return x | 397 return x |
376 # The server should return 404 when objects cannot be found. Some | 406 # The server should return 404 when objects cannot be found. Some |
377 # server implementation (ex. lfs-test-server) does not set "error" | 407 # server implementation (ex. lfs-test-server) does not set "error" |
378 # but just removes "download" from "actions". Treat that case | 408 # but just removes "download" from "actions". Treat that case |
379 # as the same as 404 error. | 409 # as the same as 404 error. |
380 if b'error' not in response: | 410 if b'error' not in response: |
381 if (action == b'download' | 411 if action == b'download' and action not in response.get( |
382 and action not in response.get(b'actions', [])): | 412 b'actions', [] |
413 ): | |
383 code = 404 | 414 code = 404 |
384 else: | 415 else: |
385 continue | 416 continue |
386 else: | 417 else: |
387 # An error dict without a code doesn't make much sense, so | 418 # An error dict without a code doesn't make much sense, so |
397 410: b'The object was removed by the owner', | 428 410: b'The object was removed by the owner', |
398 422: b'Validation error', | 429 422: b'Validation error', |
399 500: b'Internal server error', | 430 500: b'Internal server error', |
400 } | 431 } |
401 msg = errors.get(code, b'status code %d' % code) | 432 msg = errors.get(code, b'status code %d' % code) |
402 raise LfsRemoteError(_(b'LFS server error for "%s": %s') | 433 raise LfsRemoteError( |
403 % (filename, msg)) | 434 _(b'LFS server error for "%s": %s') % (filename, msg) |
435 ) | |
404 else: | 436 else: |
405 raise LfsRemoteError( | 437 raise LfsRemoteError( |
406 _(b'LFS server error. Unsolicited response for oid %s') | 438 _(b'LFS server error. Unsolicited response for oid %s') |
407 % response[b'oid']) | 439 % response[b'oid'] |
440 ) | |
408 | 441 |
409 def _extractobjects(self, response, pointers, action): | 442 def _extractobjects(self, response, pointers, action): |
410 """extract objects from response of the batch API | 443 """extract objects from response of the batch API |
411 | 444 |
412 response: parsed JSON object returned by batch API | 445 response: parsed JSON object returned by batch API |
417 objects = response.get(b'objects', []) | 450 objects = response.get(b'objects', []) |
418 self._checkforservererror(pointers, objects, action) | 451 self._checkforservererror(pointers, objects, action) |
419 | 452 |
420 # Filter objects with given action. Practically, this skips uploading | 453 # Filter objects with given action. Practically, this skips uploading |
421 # objects which exist in the server. | 454 # objects which exist in the server. |
422 filteredobjects = [o for o in objects | 455 filteredobjects = [ |
423 if action in o.get(b'actions', [])] | 456 o for o in objects if action in o.get(b'actions', []) |
457 ] | |
424 | 458 |
425 return filteredobjects | 459 return filteredobjects |
426 | 460 |
427 def _basictransfer(self, obj, action, localstore): | 461 def _basictransfer(self, obj, action, localstore): |
428 """Download or upload a single object using basic transfer protocol | 462 """Download or upload a single object using basic transfer protocol |
440 | 474 |
441 request = util.urlreq.request(pycompat.strurl(href)) | 475 request = util.urlreq.request(pycompat.strurl(href)) |
442 if action == b'upload': | 476 if action == b'upload': |
443 # If uploading blobs, read data from local blobstore. | 477 # If uploading blobs, read data from local blobstore. |
444 if not localstore.verify(oid): | 478 if not localstore.verify(oid): |
445 raise error.Abort(_(b'detected corrupt lfs object: %s') % oid, | 479 raise error.Abort( |
446 hint=_(b'run hg verify')) | 480 _(b'detected corrupt lfs object: %s') % oid, |
481 hint=_(b'run hg verify'), | |
482 ) | |
447 request.data = filewithprogress(localstore.open(oid), None) | 483 request.data = filewithprogress(localstore.open(oid), None) |
448 request.get_method = lambda: r'PUT' | 484 request.get_method = lambda: r'PUT' |
449 request.add_header(r'Content-Type', r'application/octet-stream') | 485 request.add_header(r'Content-Type', r'application/octet-stream') |
450 request.add_header(r'Content-Length', len(request.data)) | 486 request.add_header(r'Content-Length', len(request.data)) |
451 | 487 |
459 if self.ui.debugflag: | 495 if self.ui.debugflag: |
460 ui.debug(b'Status: %d\n' % req.status) | 496 ui.debug(b'Status: %d\n' % req.status) |
461 # lfs-test-server and hg serve return headers in different | 497 # lfs-test-server and hg serve return headers in different |
462 # order | 498 # order |
463 headers = pycompat.bytestr(req.info()).strip() | 499 headers = pycompat.bytestr(req.info()).strip() |
464 ui.debug(b'%s\n' | 500 ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
465 % b'\n'.join(sorted(headers.splitlines()))) | |
466 | 501 |
467 if action == b'download': | 502 if action == b'download': |
468 # If downloading blobs, store downloaded data to local | 503 # If downloading blobs, store downloaded data to local |
469 # blobstore | 504 # blobstore |
470 localstore.download(oid, req) | 505 localstore.download(oid, req) |
476 response += data | 511 response += data |
477 if response: | 512 if response: |
478 ui.debug(b'lfs %s response: %s' % (action, response)) | 513 ui.debug(b'lfs %s response: %s' % (action, response)) |
479 except util.urlerr.httperror as ex: | 514 except util.urlerr.httperror as ex: |
480 if self.ui.debugflag: | 515 if self.ui.debugflag: |
481 self.ui.debug(b'%s: %s\n' % (oid, ex.read())) # XXX: also bytes? | 516 self.ui.debug( |
482 raise LfsRemoteError(_(b'LFS HTTP error: %s (oid=%s, action=%s)') | 517 b'%s: %s\n' % (oid, ex.read()) |
483 % (stringutil.forcebytestr(ex), oid, action)) | 518 ) # XXX: also bytes? |
519 raise LfsRemoteError( | |
520 _(b'LFS HTTP error: %s (oid=%s, action=%s)') | |
521 % (stringutil.forcebytestr(ex), oid, action) | |
522 ) | |
484 except util.urlerr.urlerror as ex: | 523 except util.urlerr.urlerror as ex: |
485 hint = (_(b'attempted connection to %s') | 524 hint = _(b'attempted connection to %s') % pycompat.bytesurl( |
486 % pycompat.bytesurl(util.urllibcompat.getfullurl(request))) | 525 util.urllibcompat.getfullurl(request) |
487 raise LfsRemoteError(_(b'LFS error: %s') % _urlerrorreason(ex), | 526 ) |
488 hint=hint) | 527 raise LfsRemoteError( |
528 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint | |
529 ) | |
489 | 530 |
490 def _batch(self, pointers, localstore, action): | 531 def _batch(self, pointers, localstore, action): |
491 if action not in [b'upload', b'download']: | 532 if action not in [b'upload', b'download']: |
492 raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action) | 533 raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action) |
493 | 534 |
495 objects = self._extractobjects(response, pointers, action) | 536 objects = self._extractobjects(response, pointers, action) |
496 total = sum(x.get(b'size', 0) for x in objects) | 537 total = sum(x.get(b'size', 0) for x in objects) |
497 sizes = {} | 538 sizes = {} |
498 for obj in objects: | 539 for obj in objects: |
499 sizes[obj.get(b'oid')] = obj.get(b'size', 0) | 540 sizes[obj.get(b'oid')] = obj.get(b'size', 0) |
500 topic = {b'upload': _(b'lfs uploading'), | 541 topic = { |
501 b'download': _(b'lfs downloading')}[action] | 542 b'upload': _(b'lfs uploading'), |
543 b'download': _(b'lfs downloading'), | |
544 }[action] | |
502 if len(objects) > 1: | 545 if len(objects) > 1: |
503 self.ui.note(_(b'lfs: need to transfer %d objects (%s)\n') | 546 self.ui.note( |
504 % (len(objects), util.bytecount(total))) | 547 _(b'lfs: need to transfer %d objects (%s)\n') |
548 % (len(objects), util.bytecount(total)) | |
549 ) | |
505 | 550 |
506 def transfer(chunk): | 551 def transfer(chunk): |
507 for obj in chunk: | 552 for obj in chunk: |
508 objsize = obj.get(b'size', 0) | 553 objsize = obj.get(b'size', 0) |
509 if self.ui.verbose: | 554 if self.ui.verbose: |
510 if action == b'download': | 555 if action == b'download': |
511 msg = _(b'lfs: downloading %s (%s)\n') | 556 msg = _(b'lfs: downloading %s (%s)\n') |
512 elif action == b'upload': | 557 elif action == b'upload': |
513 msg = _(b'lfs: uploading %s (%s)\n') | 558 msg = _(b'lfs: uploading %s (%s)\n') |
514 self.ui.note(msg % (obj.get(b'oid'), | 559 self.ui.note( |
515 util.bytecount(objsize))) | 560 msg % (obj.get(b'oid'), util.bytecount(objsize)) |
561 ) | |
516 retry = self.retry | 562 retry = self.retry |
517 while True: | 563 while True: |
518 try: | 564 try: |
519 self._basictransfer(obj, action, localstore) | 565 self._basictransfer(obj, action, localstore) |
520 yield 1, obj.get(b'oid') | 566 yield 1, obj.get(b'oid') |
521 break | 567 break |
522 except socket.error as ex: | 568 except socket.error as ex: |
523 if retry > 0: | 569 if retry > 0: |
524 self.ui.note( | 570 self.ui.note( |
525 _(b'lfs: failed: %r (remaining retry %d)\n') | 571 _(b'lfs: failed: %r (remaining retry %d)\n') |
526 % (stringutil.forcebytestr(ex), retry)) | 572 % (stringutil.forcebytestr(ex), retry) |
573 ) | |
527 retry -= 1 | 574 retry -= 1 |
528 continue | 575 continue |
529 raise | 576 raise |
530 | 577 |
531 # Until https multiplexing gets sorted out | 578 # Until https multiplexing gets sorted out |
532 if self.ui.configbool(b'experimental', b'lfs.worker-enable'): | 579 if self.ui.configbool(b'experimental', b'lfs.worker-enable'): |
533 oids = worker.worker(self.ui, 0.1, transfer, (), | 580 oids = worker.worker( |
534 sorted(objects, key=lambda o: o.get(b'oid'))) | 581 self.ui, |
582 0.1, | |
583 transfer, | |
584 (), | |
585 sorted(objects, key=lambda o: o.get(b'oid')), | |
586 ) | |
535 else: | 587 else: |
536 oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) | 588 oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) |
537 | 589 |
538 with self.ui.makeprogress(topic, total=total) as progress: | 590 with self.ui.makeprogress(topic, total=total) as progress: |
539 progress.update(0) | 591 progress.update(0) |
545 progress.update(processed) | 597 progress.update(processed) |
546 self.ui.note(_(b'lfs: processed: %s\n') % oid) | 598 self.ui.note(_(b'lfs: processed: %s\n') % oid) |
547 | 599 |
548 if blobs > 0: | 600 if blobs > 0: |
549 if action == b'upload': | 601 if action == b'upload': |
550 self.ui.status(_(b'lfs: uploaded %d files (%s)\n') | 602 self.ui.status( |
551 % (blobs, util.bytecount(processed))) | 603 _(b'lfs: uploaded %d files (%s)\n') |
604 % (blobs, util.bytecount(processed)) | |
605 ) | |
552 elif action == b'download': | 606 elif action == b'download': |
553 self.ui.status(_(b'lfs: downloaded %d files (%s)\n') | 607 self.ui.status( |
554 % (blobs, util.bytecount(processed))) | 608 _(b'lfs: downloaded %d files (%s)\n') |
609 % (blobs, util.bytecount(processed)) | |
610 ) | |
555 | 611 |
556 def __del__(self): | 612 def __del__(self): |
557 # copied from mercurial/httppeer.py | 613 # copied from mercurial/httppeer.py |
558 urlopener = getattr(self, 'urlopener', None) | 614 urlopener = getattr(self, 'urlopener', None) |
559 if urlopener: | 615 if urlopener: |
560 for h in urlopener.handlers: | 616 for h in urlopener.handlers: |
561 h.close() | 617 h.close() |
562 getattr(h, "close_all", lambda : None)() | 618 getattr(h, "close_all", lambda: None)() |
619 | |
563 | 620 |
564 class _dummyremote(object): | 621 class _dummyremote(object): |
565 """Dummy store storing blobs to temp directory.""" | 622 """Dummy store storing blobs to temp directory.""" |
566 | 623 |
567 def __init__(self, repo, url): | 624 def __init__(self, repo, url): |
577 def readbatch(self, pointers, tostore): | 634 def readbatch(self, pointers, tostore): |
578 for p in _deduplicate(pointers): | 635 for p in _deduplicate(pointers): |
579 with self.vfs(p.oid(), b'rb') as fp: | 636 with self.vfs(p.oid(), b'rb') as fp: |
580 tostore.download(p.oid(), fp) | 637 tostore.download(p.oid(), fp) |
581 | 638 |
639 | |
582 class _nullremote(object): | 640 class _nullremote(object): |
583 """Null store storing blobs to /dev/null.""" | 641 """Null store storing blobs to /dev/null.""" |
584 | 642 |
585 def __init__(self, repo, url): | 643 def __init__(self, repo, url): |
586 pass | 644 pass |
589 pass | 647 pass |
590 | 648 |
591 def readbatch(self, pointers, tostore): | 649 def readbatch(self, pointers, tostore): |
592 pass | 650 pass |
593 | 651 |
652 | |
594 class _promptremote(object): | 653 class _promptremote(object): |
595 """Prompt user to set lfs.url when accessed.""" | 654 """Prompt user to set lfs.url when accessed.""" |
596 | 655 |
597 def __init__(self, repo, url): | 656 def __init__(self, repo, url): |
598 pass | 657 pass |
603 def readbatch(self, pointers, tostore, ui=None): | 662 def readbatch(self, pointers, tostore, ui=None): |
604 self._prompt() | 663 self._prompt() |
605 | 664 |
606 def _prompt(self): | 665 def _prompt(self): |
607 raise error.Abort(_(b'lfs.url needs to be configured')) | 666 raise error.Abort(_(b'lfs.url needs to be configured')) |
667 | |
608 | 668 |
609 _storemap = { | 669 _storemap = { |
610 b'https': _gitlfsremote, | 670 b'https': _gitlfsremote, |
611 b'http': _gitlfsremote, | 671 b'http': _gitlfsremote, |
612 b'file': _dummyremote, | 672 b'file': _dummyremote, |
613 b'null': _nullremote, | 673 b'null': _nullremote, |
614 None: _promptremote, | 674 None: _promptremote, |
615 } | 675 } |
616 | 676 |
677 | |
617 def _deduplicate(pointers): | 678 def _deduplicate(pointers): |
618 """Remove any duplicate oids that exist in the list""" | 679 """Remove any duplicate oids that exist in the list""" |
619 reduced = util.sortdict() | 680 reduced = util.sortdict() |
620 for p in pointers: | 681 for p in pointers: |
621 reduced[p.oid()] = p | 682 reduced[p.oid()] = p |
622 return reduced.values() | 683 return reduced.values() |
623 | 684 |
685 | |
624 def _verify(oid, content): | 686 def _verify(oid, content): |
625 realoid = node.hex(hashlib.sha256(content).digest()) | 687 realoid = node.hex(hashlib.sha256(content).digest()) |
626 if realoid != oid: | 688 if realoid != oid: |
627 raise LfsCorruptionError(_(b'detected corrupt lfs object: %s') % oid, | 689 raise LfsCorruptionError( |
628 hint=_(b'run hg verify')) | 690 _(b'detected corrupt lfs object: %s') % oid, |
691 hint=_(b'run hg verify'), | |
692 ) | |
693 | |
629 | 694 |
630 def remote(repo, remote=None): | 695 def remote(repo, remote=None): |
631 """remotestore factory. return a store in _storemap depending on config | 696 """remotestore factory. return a store in _storemap depending on config |
632 | 697 |
633 If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to | 698 If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to |
667 scheme = url.scheme | 732 scheme = url.scheme |
668 if scheme not in _storemap: | 733 if scheme not in _storemap: |
669 raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) | 734 raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) |
670 return _storemap[scheme](repo, url) | 735 return _storemap[scheme](repo, url) |
671 | 736 |
737 | |
672 class LfsRemoteError(error.StorageError): | 738 class LfsRemoteError(error.StorageError): |
673 pass | 739 pass |
674 | 740 |
741 | |
675 class LfsCorruptionError(error.Abort): | 742 class LfsCorruptionError(error.Abort): |
676 """Raised when a corrupt blob is detected, aborting an operation | 743 """Raised when a corrupt blob is detected, aborting an operation |
677 | 744 |
678 It exists to allow specialized handling on the server side.""" | 745 It exists to allow specialized handling on the server side.""" |