Mercurial > hg
annotate hgext/lfs/blobstore.py @ 46996:8d2b62d716b0 stable 5.8rc1
packaging: extract pre-computed version when running from plain tarball
The tarballs created by setup.py are not including the usual archive
markers that `hg archive` leaves, so the rewrite of the version number
computation actually left the empty version string around. This meant
that installations from PyPI would use 0.0.0 as version string.
Differential Revision: https://phab.mercurial-scm.org/D10502
author | Joerg Sonnenberger <joerg@bec.de> |
---|---|
date | Tue, 20 Apr 2021 20:35:46 +0200 |
parents | ffd3e823a7e5 |
children | 6000f5b25c9b |
rev | line source |
---|---|
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
2 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
3 # Copyright 2017 Facebook, Inc. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
4 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
7 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
9 |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
10 import contextlib |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
11 import errno |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
12 import hashlib |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
13 import json |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
14 import os |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
15 import re |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
16 import socket |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
17 |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
18 from mercurial.i18n import _ |
43089
c59eb1560c44
py3: manually import getattr where it is needed
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43077
diff
changeset
|
19 from mercurial.pycompat import getattr |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
20 from mercurial.node import hex |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
21 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
22 from mercurial import ( |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
23 encoding, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
24 error, |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
25 httpconnection as httpconnectionmod, |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
26 pathutil, |
36601
4da09b46451e
lfs: add some bytestring wrappers in blobstore.py
Augie Fackler <augie@google.com>
parents:
36455
diff
changeset
|
27 pycompat, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
28 url as urlmod, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
29 util, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
30 vfs as vfsmod, |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
31 worker, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
32 ) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
33 |
46907
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
34 from mercurial.utils import ( |
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
35 stringutil, |
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
36 urlutil, |
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
37 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
38 |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
39 from ..largefiles import lfutil |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
40 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
41 # 64 bytes for SHA256 |
36455
9e3cb58c7ab3
py3: make sure regexes are bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35927
diff
changeset
|
42 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
43 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
44 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
45 class lfsvfs(vfsmod.vfs): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
46 def join(self, path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
47 """split the path at first two characters, like: XX/XXXXX...""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
48 if not _lfsre.match(path): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
49 raise error.ProgrammingError(b'unexpected lfs path: %s' % path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
50 return super(lfsvfs, self).join(path[0:2], path[2:]) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
51 |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
52 def walk(self, path=None, onerror=None): |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
53 """Yield (dirpath, [], oids) tuple for blobs under path |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
54 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
55 Oids only exist in the root of this vfs, so dirpath is always ''. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
56 """ |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
57 root = os.path.normpath(self.base) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
58 # when dirpath == root, dirpath[prefixlen:] becomes empty |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
59 # because len(dirpath) < prefixlen. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
60 prefixlen = len(pathutil.normasprefix(root)) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
61 oids = [] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
62 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
63 for dirpath, dirs, files in os.walk( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
64 self.reljoin(self.base, path or b''), onerror=onerror |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
65 ): |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
66 dirpath = dirpath[prefixlen:] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
67 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
68 # Silently skip unexpected files and directories |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
69 if len(dirpath) == 2: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
70 oids.extend( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
71 [dirpath + f for f in files if _lfsre.match(dirpath + f)] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
72 ) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
73 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
74 yield (b'', [], oids) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
75 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
76 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
77 class nullvfs(lfsvfs): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
78 def __init__(self): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
79 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
80 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
81 def exists(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
82 return False |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
83 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
84 def read(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
85 # store.read() calls into here if the blob doesn't exist in its |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
86 # self.vfs. Raise the same error as a normal vfs when asked to read a |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
87 # file that doesn't exist. The only difference is the full file path |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
88 # isn't available in the error. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
89 raise IOError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
90 errno.ENOENT, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
91 pycompat.sysstr(b'%s: No such file or directory' % oid), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
92 ) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
93 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
94 def walk(self, path=None, onerror=None): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
95 return (b'', [], []) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
96 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
97 def write(self, oid, data): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
98 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
99 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
100 |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
101 class lfsuploadfile(httpconnectionmod.httpsendfile): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
102 """a file-like object that supports keepalive.""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
103 |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
104 def __init__(self, ui, filename): |
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
105 super(lfsuploadfile, self).__init__(ui, filename, b'rb') |
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
106 self.read = self._data.read |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
107 |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
108 def _makeprogress(self): |
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
109 return None # progress is handled by the worker client |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
110 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
111 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
112 class local(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
113 """Local blobstore for large file contents. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
114 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
115 This blobstore is used both as a cache and as a staging area for large blobs |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
116 to be uploaded to the remote blobstore. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
117 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
118 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
119 def __init__(self, repo): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
120 fullpath = repo.svfs.join(b'lfs/objects') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
121 self.vfs = lfsvfs(fullpath) |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
122 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
123 if repo.ui.configbool(b'experimental', b'lfs.disableusercache'): |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
124 self.cachevfs = nullvfs() |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
125 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
126 usercache = lfutil._usercachedir(repo.ui, b'lfs') |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
127 self.cachevfs = lfsvfs(usercache) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
128 self.ui = repo.ui |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
129 |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
130 def open(self, oid): |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
131 """Open a read-only file descriptor to the named blob, in either the |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
132 usercache or the local store.""" |
44294
234001d22ba6
lfs: use str for the open() mode when opening a blob for py3
Matt Harbison <matt_harbison@yahoo.com>
parents:
44273
diff
changeset
|
133 return open(self.path(oid), 'rb') |
44272
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
134 |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
135 def path(self, oid): |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
136 """Build the path for the given blob ``oid``. |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
137 |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
138 If the blob exists locally, the path may point to either the usercache |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
139 or the local store. If it doesn't, it will point to the local store. |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
140 This is meant for situations where existing code that isn't LFS aware |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
141 needs to open a blob. Generally, prefer the ``open`` method on this |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
142 class. |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
143 """ |
35537
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
144 # The usercache is the most likely place to hold the file. Commit will |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
145 # write to both it and the local store, as will anything that downloads |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
146 # the blobs. However, things like clone without an update won't |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
147 # populate the local store. For an init + push of a local clone, |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
148 # the usercache is the only place it _could_ be. If not present, the |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
149 # missing file msg here will indicate the local repo, not the usercache. |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
150 if self.cachevfs.exists(oid): |
44272
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
151 return self.cachevfs.join(oid) |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
152 |
44272
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
153 return self.vfs.join(oid) |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
154 |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
155 def download(self, oid, src, content_length): |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
156 """Read the blob from the remote source in chunks, verify the content, |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
157 and write to this local blobstore.""" |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
158 sha256 = hashlib.sha256() |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
159 size = 0 |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
160 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
161 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
162 for chunk in util.filechunkiter(src, size=1048576): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
163 fp.write(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
164 sha256.update(chunk) |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
165 size += len(chunk) |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
166 |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
167 # If the server advertised a length longer than what we actually |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
168 # received, then we should expect that the server crashed while |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
169 # producing the response (but the server has no way of telling us |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
170 # that), and we really don't need to try to write the response to |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
171 # the localstore, because it's not going to match the expected. |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
172 if content_length is not None and int(content_length) != size: |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
173 msg = ( |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
174 b"Response length (%s) does not match Content-Length " |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
175 b"header (%d): likely server-side crash" |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
176 ) |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
177 raise LfsRemoteError(_(msg) % (size, int(content_length))) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
178 |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
179 realoid = hex(sha256.digest()) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
180 if realoid != oid: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
181 raise LfsCorruptionError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
182 _(b'corrupt remote lfs object: %s') % oid |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
183 ) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
184 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
185 self._linktousercache(oid) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
186 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
187 def write(self, oid, data): |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
188 """Write blob to local blobstore. |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
189 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
190 This should only be called from the filelog during a commit or similar. |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
191 As such, there is no need to verify the data. Imports from a remote |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
192 store must use ``download()`` instead.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
193 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
194 fp.write(data) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
195 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
196 self._linktousercache(oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
197 |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
198 def linkfromusercache(self, oid): |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
199 """Link blobs found in the user cache into this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
200 |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
201 The server module needs to do this when it lets the client know not to |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
202 upload the blob, to ensure it is always available in this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
203 Normally this is done implicitly when the client reads or writes the |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
204 blob, but that doesn't happen when the server tells the client that it |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
205 already has the blob. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
206 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
207 if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
208 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
209 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
210 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
211 def _linktousercache(self, oid): |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
212 # XXX: should we verify the content of the cache, and hardlink back to |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
213 # the local store on success, but truncate, write and link on failure? |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
214 if not self.cachevfs.exists(oid) and not isinstance( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
215 self.cachevfs, nullvfs |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
216 ): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
217 self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
218 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
219 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
220 def read(self, oid, verify=True): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
221 """Read blob from local blobstore.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
222 if not self.vfs.exists(oid): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
223 blob = self._read(self.cachevfs, oid, verify) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
224 |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
225 # Even if revlog will verify the content, it needs to be verified |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
226 # now before making the hardlink to avoid propagating corrupt blobs. |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
227 # Don't abort if corruption is detected, because `hg verify` will |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
228 # give more useful info about the corruption- simply don't add the |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
229 # hardlink. |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
230 if verify or hex(hashlib.sha256(blob).digest()) == oid: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
231 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
232 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
233 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
234 self.ui.note(_(b'lfs: found %s in the local lfs store\n') % oid) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
235 blob = self._read(self.vfs, oid, verify) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
236 return blob |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
237 |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
238 def _read(self, vfs, oid, verify): |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
239 """Read blob (after verifying) from the given store""" |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
240 blob = vfs.read(oid) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
241 if verify: |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
242 _verify(oid, blob) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
243 return blob |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
244 |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
245 def verify(self, oid): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
246 """Indicate whether or not the hash of the underlying file matches its |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
247 name.""" |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
248 sha256 = hashlib.sha256() |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
249 |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
250 with self.open(oid) as fp: |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
251 for chunk in util.filechunkiter(fp, size=1048576): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
252 sha256.update(chunk) |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
253 |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
254 return oid == hex(sha256.digest()) |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
255 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
256 def has(self, oid): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
257 """Returns True if the local blobstore contains the requested blob, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
258 False otherwise.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
259 return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
260 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
261 |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
262 def _urlerrorreason(urlerror): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
263 """Create a friendly message for the given URLError to be used in an |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
264 LfsRemoteError message. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
265 """ |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
266 inst = urlerror |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
267 |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
268 if isinstance(urlerror.reason, Exception): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
269 inst = urlerror.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
270 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
271 if util.safehasattr(inst, b'reason'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
272 try: # usually it is in the form (errno, strerror) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
273 reason = inst.reason.args[1] |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
274 except (AttributeError, IndexError): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
275 # it might be anything, for example a string |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
276 reason = inst.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
277 if isinstance(reason, pycompat.unicode): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
278 # SSLError of Python 2.7.9 contains a unicode |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
279 reason = encoding.unitolocal(reason) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
280 return reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
281 elif getattr(inst, "strerror", None): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
282 return encoding.strtolocal(inst.strerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
283 else: |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
284 return stringutil.forcebytestr(urlerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
285 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
286 |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
287 class lfsauthhandler(util.urlreq.basehandler): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
288 handler_order = 480 # Before HTTPDigestAuthHandler (== 490) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
289 |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
290 def http_error_401(self, req, fp, code, msg, headers): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
291 """Enforces that any authentication performed is HTTP Basic |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
292 Authentication. No authentication is also acceptable. |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
293 """ |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
294 authreq = headers.get('www-authenticate', None) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
295 if authreq: |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
296 scheme = authreq.split()[0] |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
297 |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
298 if scheme.lower() != 'basic': |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
299 msg = _(b'the server must support Basic Authentication') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
300 raise util.urlerr.httperror( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
301 req.get_full_url(), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
302 code, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
303 encoding.strfromlocal(msg), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
304 headers, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
305 fp, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
306 ) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
307 return None |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
308 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
309 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
310 class _gitlfsremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
311 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
312 ui = repo.ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
313 self.ui = ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
314 baseurl, authinfo = url.authinfo() |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
315 self.baseurl = baseurl.rstrip(b'/') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
316 useragent = repo.ui.config(b'experimental', b'lfs.user-agent') |
35440
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35439
diff
changeset
|
317 if not useragent: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
318 useragent = b'git-lfs/2.3.4 (Mercurial %s)' % util.version() |
35439
e7bb5fc4570c
lfs: add git to the User-Agent header for blob transfers
Matt Harbison <matt_harbison@yahoo.com>
parents:
35433
diff
changeset
|
319 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
320 self.urlopener.add_handler(lfsauthhandler()) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
321 self.retry = ui.configint(b'lfs', b'retry') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
322 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
323 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
324 """Batch upload from local to remote blobstore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
325 self._batch(_deduplicate(pointers), fromstore, b'upload') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
326 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
327 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
328 """Batch download from remote to local blostore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
329 self._batch(_deduplicate(pointers), tostore, b'download') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
330 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
331 def _batchrequest(self, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
332 """Get metadata about objects pointed by pointers for given action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
333 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
334 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
335 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
336 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
337 objects = [ |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
338 {'oid': pycompat.strurl(p.oid()), 'size': p.size()} |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
339 for p in pointers |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
340 ] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
341 requestdata = pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
342 json.dumps( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
343 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
344 'objects': objects, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
345 'operation': pycompat.strurl(action), |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
346 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
347 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
348 ) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
349 url = b'%s/objects/batch' % self.baseurl |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
350 batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
351 batchreq.add_header('Accept', 'application/vnd.git-lfs+json') |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
352 batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
353 try: |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
354 with contextlib.closing(self.urlopener.open(batchreq)) as rsp: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
355 rawjson = rsp.read() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
356 except util.urlerr.httperror as ex: |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
357 hints = { |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
358 400: _( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
359 b'check that lfs serving is enabled on %s and "%s" is ' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
360 b'supported' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
361 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
362 % (self.baseurl, action), |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
363 404: _(b'the "lfs.url" config may be used to override %s') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
364 % self.baseurl, |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
365 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
366 hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
367 raise LfsRemoteError( |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
368 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
369 hint=hint, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
370 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
371 except util.urlerr.urlerror as ex: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
372 hint = ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
373 _(b'the "lfs.url" config may be used to override %s') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
374 % self.baseurl |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
375 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
376 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
377 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
378 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
379 try: |
43380
579672b347d2
py3: define and use json.loads polyfill
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43091
diff
changeset
|
380 response = pycompat.json_loads(rawjson) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
381 except ValueError: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
382 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
383 _(b'LFS server returns invalid JSON: %s') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
384 % rawjson.encode("utf-8") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
385 ) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
386 |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
387 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
388 self.ui.debug(b'Status: %d\n' % rsp.status) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
389 # lfs-test-server and hg serve return headers in different order |
41431
0b636d1720a0
lfs: strip the response headers from the Batch API before printing
Matt Harbison <matt_harbison@yahoo.com>
parents:
41429
diff
changeset
|
390 headers = pycompat.bytestr(rsp.info()).strip() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
391 self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
392 |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
393 if 'objects' in response: |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
394 response['objects'] = sorted( |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
395 response['objects'], key=lambda p: p['oid'] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
396 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
397 self.ui.debug( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
398 b'%s\n' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
399 % pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
400 json.dumps( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
401 response, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
402 indent=2, |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
403 separators=('', ': '), |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
404 sort_keys=True, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
405 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
406 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
407 ) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
408 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
409 def encodestr(x): |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
410 if isinstance(x, pycompat.unicode): |
43091
127cc1f72e70
py3: stop normalizing .encode()/.decode() arguments to unicode
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43089
diff
changeset
|
411 return x.encode('utf-8') |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
412 return x |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
413 |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
414 return pycompat.rapply(encodestr, response) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
415 |
35666 | 416 def _checkforservererror(self, pointers, responses, action): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
417 """Scans errors from objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
418 |
35694
8a23082f4d93
lfs: correct documentation typo
Matt Harbison <matt_harbison@yahoo.com>
parents:
35666
diff
changeset
|
419 Raises LfsRemoteError if any objects have an error""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
420 for response in responses: |
35666 | 421 # The server should return 404 when objects cannot be found. Some |
422 # server implementation (ex. lfs-test-server) does not set "error" | |
423 # but just removes "download" from "actions". Treat that case | |
424 # as the same as 404 error. | |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
425 if b'error' not in response: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
426 if action == b'download' and action not in response.get( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
427 b'actions', [] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
428 ): |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
429 code = 404 |
35695
dd672e3d059f
lfs: raise an error if the server sends an unsolicited oid
Matt Harbison <matt_harbison@yahoo.com>
parents:
35694
diff
changeset
|
430 else: |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
431 continue |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
432 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
433 # An error dict without a code doesn't make much sense, so |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
434 # treat as a server error. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
435 code = response.get(b'error').get(b'code', 500) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
436 |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
437 ptrmap = {p.oid(): p for p in pointers} |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
438 p = ptrmap.get(response[b'oid'], None) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
439 if p: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
440 filename = getattr(p, 'filename', b'unknown') |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
441 errors = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
442 404: b'The object does not exist', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
443 410: b'The object was removed by the owner', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
444 422: b'Validation error', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
445 500: b'Internal server error', |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
446 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
447 msg = errors.get(code, b'status code %d' % code) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
448 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
449 _(b'LFS server error for "%s": %s') % (filename, msg) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
450 ) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
451 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
452 raise LfsRemoteError( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
453 _(b'LFS server error. Unsolicited response for oid %s') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
454 % response[b'oid'] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
455 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
456 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
457 def _extractobjects(self, response, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
458 """extract objects from response of the batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
459 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
460 response: parsed JSON object returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
461 return response['objects'] filtered by action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
462 raise if any object has an error |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
463 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
464 # Scan errors from objects - fail early |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
465 objects = response.get(b'objects', []) |
35666 | 466 self._checkforservererror(pointers, objects, action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
467 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
468 # Filter objects with given action. Practically, this skips uploading |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
469 # objects which exist in the server. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
470 filteredobjects = [ |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
471 o for o in objects if action in o.get(b'actions', []) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
472 ] |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
473 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
474 return filteredobjects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
475 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
476 def _basictransfer(self, obj, action, localstore): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
477 """Download or upload a single object using basic transfer protocol |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
478 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
479 obj: dict, an object description returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
480 action: string, one of ['upload', 'download'] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
481 localstore: blobstore.local |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
482 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
483 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
484 basic-transfers.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
485 """ |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
486 oid = obj[b'oid'] |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
487 href = obj[b'actions'][action].get(b'href') |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
488 headers = obj[b'actions'][action].get(b'header', {}).items() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
489 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
490 request = util.urlreq.request(pycompat.strurl(href)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
491 if action == b'upload': |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
492 # If uploading blobs, read data from local blobstore. |
37217
b00bd974eef5
lfs: drop a duplicate blob verification method
Matt Harbison <matt_harbison@yahoo.com>
parents:
37146
diff
changeset
|
493 if not localstore.verify(oid): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
494 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
495 _(b'detected corrupt lfs object: %s') % oid, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
496 hint=_(b'run hg verify'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
497 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
498 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
499 for k, v in headers: |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
500 request.add_header(pycompat.strurl(k), pycompat.strurl(v)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
501 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
502 try: |
44141
46c8f15fb2b4
lfs: move the initialization of the upload request into the try block
Matt Harbison <matt_harbison@yahoo.com>
parents:
44140
diff
changeset
|
503 if action == b'upload': |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
504 request.data = lfsuploadfile(self.ui, localstore.path(oid)) |
44141
46c8f15fb2b4
lfs: move the initialization of the upload request into the try block
Matt Harbison <matt_harbison@yahoo.com>
parents:
44140
diff
changeset
|
505 request.get_method = lambda: 'PUT' |
46c8f15fb2b4
lfs: move the initialization of the upload request into the try block
Matt Harbison <matt_harbison@yahoo.com>
parents:
44140
diff
changeset
|
506 request.add_header('Content-Type', 'application/octet-stream') |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
507 request.add_header('Content-Length', request.data.length) |
44141
46c8f15fb2b4
lfs: move the initialization of the upload request into the try block
Matt Harbison <matt_harbison@yahoo.com>
parents:
44140
diff
changeset
|
508 |
44084
84f2becbd106
lfs: rename a variable to clarify its use
Matt Harbison <matt_harbison@yahoo.com>
parents:
44077
diff
changeset
|
509 with contextlib.closing(self.urlopener.open(request)) as res: |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
510 contentlength = res.info().get(b"content-length") |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
511 ui = self.ui # Shorten debug lines |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
512 if self.ui.debugflag: |
44084
84f2becbd106
lfs: rename a variable to clarify its use
Matt Harbison <matt_harbison@yahoo.com>
parents:
44077
diff
changeset
|
513 ui.debug(b'Status: %d\n' % res.status) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
514 # lfs-test-server and hg serve return headers in different |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
515 # order |
44084
84f2becbd106
lfs: rename a variable to clarify its use
Matt Harbison <matt_harbison@yahoo.com>
parents:
44077
diff
changeset
|
516 headers = pycompat.bytestr(res.info()).strip() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
517 ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
518 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
519 if action == b'download': |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
520 # If downloading blobs, store downloaded data to local |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
521 # blobstore |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
522 localstore.download(oid, res, contentlength) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
523 else: |
44086
ffac09da7a19
lfs: avoid quadratic performance in processing server responses
Matt Harbison <matt_harbison@yahoo.com>
parents:
44085
diff
changeset
|
524 blocks = [] |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
525 while True: |
44084
84f2becbd106
lfs: rename a variable to clarify its use
Matt Harbison <matt_harbison@yahoo.com>
parents:
44077
diff
changeset
|
526 data = res.read(1048576) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
527 if not data: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
528 break |
44086
ffac09da7a19
lfs: avoid quadratic performance in processing server responses
Matt Harbison <matt_harbison@yahoo.com>
parents:
44085
diff
changeset
|
529 blocks.append(data) |
ffac09da7a19
lfs: avoid quadratic performance in processing server responses
Matt Harbison <matt_harbison@yahoo.com>
parents:
44085
diff
changeset
|
530 |
ffac09da7a19
lfs: avoid quadratic performance in processing server responses
Matt Harbison <matt_harbison@yahoo.com>
parents:
44085
diff
changeset
|
531 response = b"".join(blocks) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
532 if response: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
533 ui.debug(b'lfs %s response: %s' % (action, response)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
534 except util.urlerr.httperror as ex: |
35734
b4e1d0654736
lfs: dump the full response on httperror in debug mode
Matt Harbison <matt_harbison@yahoo.com>
parents:
35733
diff
changeset
|
535 if self.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
536 self.ui.debug( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
537 b'%s: %s\n' % (oid, ex.read()) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
538 ) # XXX: also bytes? |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
539 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
540 _(b'LFS HTTP error: %s (oid=%s, action=%s)') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
541 % (stringutil.forcebytestr(ex), oid, action) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
542 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
543 except util.urlerr.urlerror as ex: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
544 hint = _(b'attempted connection to %s') % pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
545 util.urllibcompat.getfullurl(request) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
546 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
547 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
548 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
549 ) |
44139
2ad4e8aefcf4
lfs: explicitly close the file handle for the blob being uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
44138
diff
changeset
|
550 finally: |
2ad4e8aefcf4
lfs: explicitly close the file handle for the blob being uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
44138
diff
changeset
|
551 if request.data: |
2ad4e8aefcf4
lfs: explicitly close the file handle for the blob being uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
44138
diff
changeset
|
552 request.data.close() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
553 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
554 def _batch(self, pointers, localstore, action): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
555 if action not in [b'upload', b'download']: |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
556 raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
557 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
558 response = self._batchrequest(pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
559 objects = self._extractobjects(response, pointers, action) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
560 total = sum(x.get(b'size', 0) for x in objects) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
561 sizes = {} |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
562 for obj in objects: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
563 sizes[obj.get(b'oid')] = obj.get(b'size', 0) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
564 topic = { |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
565 b'upload': _(b'lfs uploading'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
566 b'download': _(b'lfs downloading'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
567 }[action] |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
568 if len(objects) > 1: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
569 self.ui.note( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
570 _(b'lfs: need to transfer %d objects (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
571 % (len(objects), util.bytecount(total)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
572 ) |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
573 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
574 def transfer(chunk): |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
575 for obj in chunk: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
576 objsize = obj.get(b'size', 0) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
577 if self.ui.verbose: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
578 if action == b'download': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
579 msg = _(b'lfs: downloading %s (%s)\n') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
580 elif action == b'upload': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
581 msg = _(b'lfs: uploading %s (%s)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
582 self.ui.note( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
583 msg % (obj.get(b'oid'), util.bytecount(objsize)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
584 ) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
585 retry = self.retry |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
586 while True: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
587 try: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
588 self._basictransfer(obj, action, localstore) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
589 yield 1, obj.get(b'oid') |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
590 break |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
591 except socket.error as ex: |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
592 if retry > 0: |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
593 self.ui.note( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
594 _(b'lfs: failed: %r (remaining retry %d)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
595 % (stringutil.forcebytestr(ex), retry) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
596 ) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
597 retry -= 1 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
598 continue |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
599 raise |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
600 |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
601 # Until https multiplexing gets sorted out |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
602 if self.ui.configbool(b'experimental', b'lfs.worker-enable'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
603 oids = worker.worker( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
604 self.ui, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
605 0.1, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
606 transfer, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
607 (), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
608 sorted(objects, key=lambda o: o.get(b'oid')), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
609 ) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
610 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
611 oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
612 |
44077
05881d002cb2
lfs: add "bytes" as the unit to the upload/download progress bar
Matt Harbison <matt_harbison@yahoo.com>
parents:
43506
diff
changeset
|
613 with self.ui.makeprogress( |
05881d002cb2
lfs: add "bytes" as the unit to the upload/download progress bar
Matt Harbison <matt_harbison@yahoo.com>
parents:
43506
diff
changeset
|
614 topic, unit=_(b"bytes"), total=total |
05881d002cb2
lfs: add "bytes" as the unit to the upload/download progress bar
Matt Harbison <matt_harbison@yahoo.com>
parents:
43506
diff
changeset
|
615 ) as progress: |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
616 progress.update(0) |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
617 processed = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
618 blobs = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
619 for _one, oid in oids: |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
620 processed += sizes[oid] |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
621 blobs += 1 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
622 progress.update(processed) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
623 self.ui.note(_(b'lfs: processed: %s\n') % oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
624 |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
625 if blobs > 0: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
626 if action == b'upload': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
627 self.ui.status( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
628 _(b'lfs: uploaded %d files (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
629 % (blobs, util.bytecount(processed)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
630 ) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
631 elif action == b'download': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
632 self.ui.status( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
633 _(b'lfs: downloaded %d files (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
634 % (blobs, util.bytecount(processed)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
635 ) |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
636 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
637 def __del__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
638 # copied from mercurial/httppeer.py |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
639 urlopener = getattr(self, 'urlopener', None) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
640 if urlopener: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
641 for h in urlopener.handlers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
642 h.close() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
643 getattr(h, "close_all", lambda: None)() |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
644 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
645 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
646 class _dummyremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
647 """Dummy store storing blobs to temp directory.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
648 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
649 def __init__(self, repo, url): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
650 fullpath = repo.vfs.join(b'lfs', url.path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
651 self.vfs = lfsvfs(fullpath) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
652 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
653 def writebatch(self, pointers, fromstore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
654 for p in _deduplicate(pointers): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
655 content = fromstore.read(p.oid(), verify=True) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
656 with self.vfs(p.oid(), b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
657 fp.write(content) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
658 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
659 def readbatch(self, pointers, tostore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
660 for p in _deduplicate(pointers): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
661 with self.vfs(p.oid(), b'rb') as fp: |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
662 tostore.download(p.oid(), fp, None) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
663 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
664 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
665 class _nullremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
666 """Null store storing blobs to /dev/null.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
667 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
668 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
669 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
670 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
671 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
672 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
673 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
674 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
675 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
676 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
677 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
678 class _promptremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
679 """Prompt user to set lfs.url when accessed.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
680 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
681 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
682 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
683 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
684 def writebatch(self, pointers, fromstore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
685 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
686 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
687 def readbatch(self, pointers, tostore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
688 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
689 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
690 def _prompt(self): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
691 raise error.Abort(_(b'lfs.url needs to be configured')) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
692 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
693 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
694 _storemap = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
695 b'https': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
696 b'http': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
697 b'file': _dummyremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
698 b'null': _nullremote, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
699 None: _promptremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
700 } |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
701 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
702 |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
703 def _deduplicate(pointers): |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
704 """Remove any duplicate oids that exist in the list""" |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
705 reduced = util.sortdict() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
706 for p in pointers: |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
707 reduced[p.oid()] = p |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
708 return reduced.values() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
709 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
710 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
711 def _verify(oid, content): |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
712 realoid = hex(hashlib.sha256(content).digest()) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
713 if realoid != oid: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
714 raise LfsCorruptionError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
715 _(b'detected corrupt lfs object: %s') % oid, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
716 hint=_(b'run hg verify'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
717 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
718 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
719 |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
720 def remote(repo, remote=None): |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
721 """remotestore factory. return a store in _storemap depending on config |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
722 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
723 If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
724 infer the endpoint, based on the remote repository using the same path |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
725 adjustments as git. As an extension, 'http' is supported as well so that |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
726 ``hg serve`` works out of the box. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
727 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
728 https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
729 """ |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
730 lfsurl = repo.ui.config(b'lfs', b'url') |
46907
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
731 url = urlutil.url(lfsurl or b'') |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
732 if lfsurl is None: |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
733 if remote: |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
734 path = remote |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
735 elif util.safehasattr(repo, b'_subtoppath'): |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
736 # The pull command sets this during the optional update phase, which |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
737 # tells exactly where the pull originated, whether 'paths.default' |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
738 # or explicit. |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
739 path = repo._subtoppath |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
740 else: |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
741 # TODO: investigate 'paths.remote:lfsurl' style path customization, |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
742 # and fall back to inferring from 'paths.remote' if unspecified. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
743 path = repo.ui.config(b'paths', b'default') or b'' |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
744 |
46907
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
745 defaulturl = urlutil.url(path) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
746 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
747 # TODO: support local paths as well. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
748 # TODO: consider the ssh -> https transformation that git applies |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
749 if defaulturl.scheme in (b'http', b'https'): |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
750 if defaulturl.path and defaulturl.path[:-1] != b'/': |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
751 defaulturl.path += b'/' |
37691
d241e6632669
lfs: fix the inferred remote store path when using a --prefix
Matt Harbison <matt_harbison@yahoo.com>
parents:
37565
diff
changeset
|
752 defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs' |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
753 |
46907
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
754 url = urlutil.url(bytes(defaulturl)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
755 repo.ui.note(_(b'lfs: assuming remote store: %s\n') % url) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
756 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
757 scheme = url.scheme |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
758 if scheme not in _storemap: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
759 raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
760 return _storemap[scheme](repo, url) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
761 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
762 |
39777
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39457
diff
changeset
|
763 class LfsRemoteError(error.StorageError): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
764 pass |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
765 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
766 |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
767 class LfsCorruptionError(error.Abort): |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
768 """Raised when a corrupt blob is detected, aborting an operation |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
769 |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
770 It exists to allow specialized handling on the server side.""" |