Mercurial > hg
annotate hgext/lfs/blobstore.py @ 50308:cab3defe6d3b stable
transaction: raise on backup restoration error
A few line above, similar errors in the truncation code result in raising the
associated exception. We should do the same here.
This means the transaction recover is more strict now, which might be a problem
when running `hg recover` in a share different from the one where the
transaction fails. However this has always been a problem and need to be be
addressed independently.
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Wed, 15 Mar 2023 12:13:08 +0100 |
parents | 18282cf18aa2 |
children | 6543469a351e |
rev | line source |
---|---|
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
2 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
3 # Copyright 2017 Facebook, Inc. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
4 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
7 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
8 |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
9 import contextlib |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
10 import errno |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
11 import hashlib |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
12 import json |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
13 import os |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
14 import re |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
15 import socket |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
16 |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
17 from mercurial.i18n import _ |
43089
c59eb1560c44
py3: manually import getattr where it is needed
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43077
diff
changeset
|
18 from mercurial.pycompat import getattr |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
19 from mercurial.node import hex |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
20 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
21 from mercurial import ( |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
22 encoding, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
23 error, |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
24 httpconnection as httpconnectionmod, |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
25 pathutil, |
36601
4da09b46451e
lfs: add some bytestring wrappers in blobstore.py
Augie Fackler <augie@google.com>
parents:
36455
diff
changeset
|
26 pycompat, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
27 url as urlmod, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
28 util, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
29 vfs as vfsmod, |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
30 worker, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
31 ) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
32 |
46907
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
33 from mercurial.utils import ( |
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
34 stringutil, |
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
35 urlutil, |
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
36 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
37 |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
38 from ..largefiles import lfutil |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
39 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
40 # 64 bytes for SHA256 |
36455
9e3cb58c7ab3
py3: make sure regexes are bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35927
diff
changeset
|
41 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
42 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
43 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
44 class lfsvfs(vfsmod.vfs): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
45 def join(self, path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
46 """split the path at first two characters, like: XX/XXXXX...""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
47 if not _lfsre.match(path): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
48 raise error.ProgrammingError(b'unexpected lfs path: %s' % path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
49 return super(lfsvfs, self).join(path[0:2], path[2:]) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
50 |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
51 def walk(self, path=None, onerror=None): |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
52 """Yield (dirpath, [], oids) tuple for blobs under path |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
53 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
54 Oids only exist in the root of this vfs, so dirpath is always ''. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
55 """ |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
56 root = os.path.normpath(self.base) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
57 # when dirpath == root, dirpath[prefixlen:] becomes empty |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
58 # because len(dirpath) < prefixlen. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
59 prefixlen = len(pathutil.normasprefix(root)) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
60 oids = [] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
61 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
62 for dirpath, dirs, files in os.walk( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
63 self.reljoin(self.base, path or b''), onerror=onerror |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
64 ): |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
65 dirpath = dirpath[prefixlen:] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
66 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
67 # Silently skip unexpected files and directories |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
68 if len(dirpath) == 2: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
69 oids.extend( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
70 [dirpath + f for f in files if _lfsre.match(dirpath + f)] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
71 ) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
72 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
73 yield (b'', [], oids) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
74 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
75 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
76 class nullvfs(lfsvfs): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
77 def __init__(self): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
78 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
79 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
80 def exists(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
81 return False |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
82 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
83 def read(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
84 # store.read() calls into here if the blob doesn't exist in its |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
85 # self.vfs. Raise the same error as a normal vfs when asked to read a |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
86 # file that doesn't exist. The only difference is the full file path |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
87 # isn't available in the error. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
88 raise IOError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
89 errno.ENOENT, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
90 pycompat.sysstr(b'%s: No such file or directory' % oid), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
91 ) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
92 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
93 def walk(self, path=None, onerror=None): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
94 return (b'', [], []) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
95 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
96 def write(self, oid, data): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
97 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
98 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
99 |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
100 class lfsuploadfile(httpconnectionmod.httpsendfile): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
101 """a file-like object that supports keepalive.""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
102 |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
103 def __init__(self, ui, filename): |
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
104 super(lfsuploadfile, self).__init__(ui, filename, b'rb') |
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
105 self.read = self._data.read |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
106 |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
107 def _makeprogress(self): |
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
108 return None # progress is handled by the worker client |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
109 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
110 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48934
diff
changeset
|
111 class local: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
112 """Local blobstore for large file contents. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
113 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
114 This blobstore is used both as a cache and as a staging area for large blobs |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
115 to be uploaded to the remote blobstore. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
116 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
117 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
118 def __init__(self, repo): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
119 fullpath = repo.svfs.join(b'lfs/objects') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
120 self.vfs = lfsvfs(fullpath) |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
121 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
122 if repo.ui.configbool(b'experimental', b'lfs.disableusercache'): |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
123 self.cachevfs = nullvfs() |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
124 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
125 usercache = lfutil._usercachedir(repo.ui, b'lfs') |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
126 self.cachevfs = lfsvfs(usercache) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
127 self.ui = repo.ui |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
128 |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
129 def open(self, oid): |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
130 """Open a read-only file descriptor to the named blob, in either the |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
131 usercache or the local store.""" |
44294
234001d22ba6
lfs: use str for the open() mode when opening a blob for py3
Matt Harbison <matt_harbison@yahoo.com>
parents:
44273
diff
changeset
|
132 return open(self.path(oid), 'rb') |
44272
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
133 |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
134 def path(self, oid): |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
135 """Build the path for the given blob ``oid``. |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
136 |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
137 If the blob exists locally, the path may point to either the usercache |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
138 or the local store. If it doesn't, it will point to the local store. |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
139 This is meant for situations where existing code that isn't LFS aware |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
140 needs to open a blob. Generally, prefer the ``open`` method on this |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
141 class. |
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
142 """ |
35537
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
143 # The usercache is the most likely place to hold the file. Commit will |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
144 # write to both it and the local store, as will anything that downloads |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
145 # the blobs. However, things like clone without an update won't |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
146 # populate the local store. For an init + push of a local clone, |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
147 # the usercache is the only place it _could_ be. If not present, the |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
148 # missing file msg here will indicate the local repo, not the usercache. |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
149 if self.cachevfs.exists(oid): |
44272
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
150 return self.cachevfs.join(oid) |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
151 |
44272
06de4a673f48
lfs: add a method to the local blobstore to convert OIDs to file paths
Matt Harbison <matt_harbison@yahoo.com>
parents:
44141
diff
changeset
|
152 return self.vfs.join(oid) |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
153 |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
154 def download(self, oid, src, content_length): |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
155 """Read the blob from the remote source in chunks, verify the content, |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
156 and write to this local blobstore.""" |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
157 sha256 = hashlib.sha256() |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
158 size = 0 |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
159 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
160 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
161 for chunk in util.filechunkiter(src, size=1048576): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
162 fp.write(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
163 sha256.update(chunk) |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
164 size += len(chunk) |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
165 |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
166 # If the server advertised a length longer than what we actually |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
167 # received, then we should expect that the server crashed while |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
168 # producing the response (but the server has no way of telling us |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
169 # that), and we really don't need to try to write the response to |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
170 # the localstore, because it's not going to match the expected. |
49532
250d9c8aaf10
lfs: improve an exception message for blob corruption detected on transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
171 # The server also uses this method to store data uploaded by the |
250d9c8aaf10
lfs: improve an exception message for blob corruption detected on transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
172 # client, so if this happens on the server side, it's possible |
250d9c8aaf10
lfs: improve an exception message for blob corruption detected on transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
173 # that the client crashed or an antivirus interfered with the |
250d9c8aaf10
lfs: improve an exception message for blob corruption detected on transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
174 # upload. |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
175 if content_length is not None and int(content_length) != size: |
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
176 msg = ( |
49526
192949b68159
lfs: fix interpolation of int and %s in an exception case
Matt Harbison <matt_harbison@yahoo.com>
parents:
48946
diff
changeset
|
177 b"Response length (%d) does not match Content-Length " |
49532
250d9c8aaf10
lfs: improve an exception message for blob corruption detected on transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
178 b"header (%d) for %s" |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
179 ) |
49532
250d9c8aaf10
lfs: improve an exception message for blob corruption detected on transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
180 raise LfsRemoteError(_(msg) % (size, int(content_length), oid)) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
181 |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
182 realoid = hex(sha256.digest()) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
183 if realoid != oid: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
184 raise LfsCorruptionError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
185 _(b'corrupt remote lfs object: %s') % oid |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
186 ) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
187 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
188 self._linktousercache(oid) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
189 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
190 def write(self, oid, data): |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
191 """Write blob to local blobstore. |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
192 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
193 This should only be called from the filelog during a commit or similar. |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
194 As such, there is no need to verify the data. Imports from a remote |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
195 store must use ``download()`` instead.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
196 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
197 fp.write(data) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
198 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
199 self._linktousercache(oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
200 |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
201 def linkfromusercache(self, oid): |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
202 """Link blobs found in the user cache into this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
203 |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
204 The server module needs to do this when it lets the client know not to |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
205 upload the blob, to ensure it is always available in this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
206 Normally this is done implicitly when the client reads or writes the |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
207 blob, but that doesn't happen when the server tells the client that it |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
208 already has the blob. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
209 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
210 if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
211 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
212 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
213 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
214 def _linktousercache(self, oid): |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
215 # XXX: should we verify the content of the cache, and hardlink back to |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
216 # the local store on success, but truncate, write and link on failure? |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
217 if not self.cachevfs.exists(oid) and not isinstance( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
218 self.cachevfs, nullvfs |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
219 ): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
220 self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
221 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
222 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
223 def read(self, oid, verify=True): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
224 """Read blob from local blobstore.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
225 if not self.vfs.exists(oid): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
226 blob = self._read(self.cachevfs, oid, verify) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
227 |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
228 # Even if revlog will verify the content, it needs to be verified |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
229 # now before making the hardlink to avoid propagating corrupt blobs. |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
230 # Don't abort if corruption is detected, because `hg verify` will |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
231 # give more useful info about the corruption- simply don't add the |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
232 # hardlink. |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
233 if verify or hex(hashlib.sha256(blob).digest()) == oid: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
234 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
235 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
236 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
237 self.ui.note(_(b'lfs: found %s in the local lfs store\n') % oid) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
238 blob = self._read(self.vfs, oid, verify) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
239 return blob |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
240 |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
241 def _read(self, vfs, oid, verify): |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
242 """Read blob (after verifying) from the given store""" |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
243 blob = vfs.read(oid) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
244 if verify: |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
245 _verify(oid, blob) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
246 return blob |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
247 |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
248 def verify(self, oid): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
249 """Indicate whether or not the hash of the underlying file matches its |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
250 name.""" |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
251 sha256 = hashlib.sha256() |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
252 |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
253 with self.open(oid) as fp: |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
254 for chunk in util.filechunkiter(fp, size=1048576): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
255 sha256.update(chunk) |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
256 |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
257 return oid == hex(sha256.digest()) |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
258 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
259 def has(self, oid): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
260 """Returns True if the local blobstore contains the requested blob, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
261 False otherwise.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
262 return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
263 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
264 |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
265 def _urlerrorreason(urlerror): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
266 """Create a friendly message for the given URLError to be used in an |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
267 LfsRemoteError message. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
268 """ |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
269 inst = urlerror |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
270 |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
271 if isinstance(urlerror.reason, Exception): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
272 inst = urlerror.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
273 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
274 if util.safehasattr(inst, b'reason'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
275 try: # usually it is in the form (errno, strerror) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
276 reason = inst.reason.args[1] |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
277 except (AttributeError, IndexError): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
278 # it might be anything, for example a string |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
279 reason = inst.reason |
48934
06de08b36c82
py3: use str instead of pycompat.unicode
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48875
diff
changeset
|
280 if isinstance(reason, str): |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
281 # SSLError of Python 2.7.9 contains a unicode |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
282 reason = encoding.unitolocal(reason) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
283 return reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
284 elif getattr(inst, "strerror", None): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
285 return encoding.strtolocal(inst.strerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
286 else: |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
287 return stringutil.forcebytestr(urlerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
288 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
289 |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
290 class lfsauthhandler(util.urlreq.basehandler): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
291 handler_order = 480 # Before HTTPDigestAuthHandler (== 490) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
292 |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
293 def http_error_401(self, req, fp, code, msg, headers): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
294 """Enforces that any authentication performed is HTTP Basic |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
295 Authentication. No authentication is also acceptable. |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
296 """ |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
297 authreq = headers.get('www-authenticate', None) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
298 if authreq: |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
299 scheme = authreq.split()[0] |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
300 |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
301 if scheme.lower() != 'basic': |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
302 msg = _(b'the server must support Basic Authentication') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
303 raise util.urlerr.httperror( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
304 req.get_full_url(), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
305 code, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
306 encoding.strfromlocal(msg), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
307 headers, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
308 fp, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
309 ) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
310 return None |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
311 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
312 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48934
diff
changeset
|
313 class _gitlfsremote: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
314 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
315 ui = repo.ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
316 self.ui = ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
317 baseurl, authinfo = url.authinfo() |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
318 self.baseurl = baseurl.rstrip(b'/') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
319 useragent = repo.ui.config(b'experimental', b'lfs.user-agent') |
35440
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35439
diff
changeset
|
320 if not useragent: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
321 useragent = b'git-lfs/2.3.4 (Mercurial %s)' % util.version() |
35439
e7bb5fc4570c
lfs: add git to the User-Agent header for blob transfers
Matt Harbison <matt_harbison@yahoo.com>
parents:
35433
diff
changeset
|
322 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
323 self.urlopener.add_handler(lfsauthhandler()) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
324 self.retry = ui.configint(b'lfs', b'retry') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
325 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
326 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
327 """Batch upload from local to remote blobstore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
328 self._batch(_deduplicate(pointers), fromstore, b'upload') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
329 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
330 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
331 """Batch download from remote to local blostore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
332 self._batch(_deduplicate(pointers), tostore, b'download') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
333 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
334 def _batchrequest(self, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
335 """Get metadata about objects pointed by pointers for given action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
336 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
337 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
338 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
339 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
340 objects = [ |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
341 {'oid': pycompat.strurl(p.oid()), 'size': p.size()} |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
342 for p in pointers |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
343 ] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
344 requestdata = pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
345 json.dumps( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
346 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
347 'objects': objects, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
348 'operation': pycompat.strurl(action), |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44294
diff
changeset
|
349 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
350 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
351 ) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
352 url = b'%s/objects/batch' % self.baseurl |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
353 batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
354 batchreq.add_header('Accept', 'application/vnd.git-lfs+json') |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
355 batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
356 try: |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
357 with contextlib.closing(self.urlopener.open(batchreq)) as rsp: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
358 rawjson = rsp.read() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
359 except util.urlerr.httperror as ex: |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
360 hints = { |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
361 400: _( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
362 b'check that lfs serving is enabled on %s and "%s" is ' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
363 b'supported' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
364 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
365 % (self.baseurl, action), |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
366 404: _(b'the "lfs.url" config may be used to override %s') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
367 % self.baseurl, |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
368 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
369 hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
370 raise LfsRemoteError( |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
371 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
372 hint=hint, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
373 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
374 except util.urlerr.urlerror as ex: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
375 hint = ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
376 _(b'the "lfs.url" config may be used to override %s') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
377 % self.baseurl |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
378 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
379 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
380 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
381 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
382 try: |
43380
579672b347d2
py3: define and use json.loads polyfill
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43091
diff
changeset
|
383 response = pycompat.json_loads(rawjson) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
384 except ValueError: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
385 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
386 _(b'LFS server returns invalid JSON: %s') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
387 % rawjson.encode("utf-8") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
388 ) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
389 |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
390 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
391 self.ui.debug(b'Status: %d\n' % rsp.status) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
392 # lfs-test-server and hg serve return headers in different order |
41431
0b636d1720a0
lfs: strip the response headers from the Batch API before printing
Matt Harbison <matt_harbison@yahoo.com>
parents:
41429
diff
changeset
|
393 headers = pycompat.bytestr(rsp.info()).strip() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
394 self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
395 |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
396 if 'objects' in response: |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
397 response['objects'] = sorted( |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
398 response['objects'], key=lambda p: p['oid'] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
399 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
400 self.ui.debug( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
401 b'%s\n' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
402 % pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
403 json.dumps( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
404 response, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
405 indent=2, |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43380
diff
changeset
|
406 separators=('', ': '), |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
407 sort_keys=True, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
408 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
409 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
410 ) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
411 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
412 def encodestr(x): |
48934
06de08b36c82
py3: use str instead of pycompat.unicode
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48875
diff
changeset
|
413 if isinstance(x, str): |
43091
127cc1f72e70
py3: stop normalizing .encode()/.decode() arguments to unicode
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43089
diff
changeset
|
414 return x.encode('utf-8') |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
415 return x |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
416 |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
417 return pycompat.rapply(encodestr, response) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
418 |
35666 | 419 def _checkforservererror(self, pointers, responses, action): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
420 """Scans errors from objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
421 |
35694
8a23082f4d93
lfs: correct documentation typo
Matt Harbison <matt_harbison@yahoo.com>
parents:
35666
diff
changeset
|
422 Raises LfsRemoteError if any objects have an error""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
423 for response in responses: |
35666 | 424 # The server should return 404 when objects cannot be found. Some |
425 # server implementation (ex. lfs-test-server) does not set "error" | |
426 # but just removes "download" from "actions". Treat that case | |
427 # as the same as 404 error. | |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
428 if b'error' not in response: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
429 if action == b'download' and action not in response.get( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
430 b'actions', [] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
431 ): |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
432 code = 404 |
35695
dd672e3d059f
lfs: raise an error if the server sends an unsolicited oid
Matt Harbison <matt_harbison@yahoo.com>
parents:
35694
diff
changeset
|
433 else: |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
434 continue |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
435 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
436 # An error dict without a code doesn't make much sense, so |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
437 # treat as a server error. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
438 code = response.get(b'error').get(b'code', 500) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
439 |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
440 ptrmap = {p.oid(): p for p in pointers} |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
441 p = ptrmap.get(response[b'oid'], None) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
442 if p: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
443 filename = getattr(p, 'filename', b'unknown') |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
444 errors = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
445 404: b'The object does not exist', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
446 410: b'The object was removed by the owner', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
447 422: b'Validation error', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
448 500: b'Internal server error', |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
449 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
450 msg = errors.get(code, b'status code %d' % code) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
451 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
452 _(b'LFS server error for "%s": %s') % (filename, msg) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
453 ) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
454 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
455 raise LfsRemoteError( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
456 _(b'LFS server error. Unsolicited response for oid %s') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
457 % response[b'oid'] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
458 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
459 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
460 def _extractobjects(self, response, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
461 """extract objects from response of the batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
462 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
463 response: parsed JSON object returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
464 return response['objects'] filtered by action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
465 raise if any object has an error |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
466 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
467 # Scan errors from objects - fail early |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
468 objects = response.get(b'objects', []) |
35666 | 469 self._checkforservererror(pointers, objects, action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
470 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
471 # Filter objects with given action. Practically, this skips uploading |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
472 # objects which exist in the server. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
473 filteredobjects = [ |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
474 o for o in objects if action in o.get(b'actions', []) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
475 ] |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
476 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
477 return filteredobjects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
478 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
479 def _basictransfer(self, obj, action, localstore): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
480 """Download or upload a single object using basic transfer protocol |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
481 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
482 obj: dict, an object description returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
483 action: string, one of ['upload', 'download'] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
484 localstore: blobstore.local |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
485 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
486 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
487 basic-transfers.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
488 """ |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
489 oid = obj[b'oid'] |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
490 href = obj[b'actions'][action].get(b'href') |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
491 headers = obj[b'actions'][action].get(b'header', {}).items() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
492 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
493 request = util.urlreq.request(pycompat.strurl(href)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
494 if action == b'upload': |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
495 # If uploading blobs, read data from local blobstore. |
37217
b00bd974eef5
lfs: drop a duplicate blob verification method
Matt Harbison <matt_harbison@yahoo.com>
parents:
37146
diff
changeset
|
496 if not localstore.verify(oid): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
497 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
498 _(b'detected corrupt lfs object: %s') % oid, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
499 hint=_(b'run hg verify'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
500 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
501 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
502 for k, v in headers: |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
503 request.add_header(pycompat.strurl(k), pycompat.strurl(v)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
504 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
505 try: |
44141
46c8f15fb2b4
lfs: move the initialization of the upload request into the try block
Matt Harbison <matt_harbison@yahoo.com>
parents:
44140
diff
changeset
|
506 if action == b'upload': |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
507 request.data = lfsuploadfile(self.ui, localstore.path(oid)) |
44141
46c8f15fb2b4
lfs: move the initialization of the upload request into the try block
Matt Harbison <matt_harbison@yahoo.com>
parents:
44140
diff
changeset
|
508 request.get_method = lambda: 'PUT' |
46c8f15fb2b4
lfs: move the initialization of the upload request into the try block
Matt Harbison <matt_harbison@yahoo.com>
parents:
44140
diff
changeset
|
509 request.add_header('Content-Type', 'application/octet-stream') |
44273
43eea17ae7b3
lfs: fix the stall and corruption issue when concurrently uploading blobs
Matt Harbison <matt_harbison@yahoo.com>
parents:
44272
diff
changeset
|
510 request.add_header('Content-Length', request.data.length) |
44141
46c8f15fb2b4
lfs: move the initialization of the upload request into the try block
Matt Harbison <matt_harbison@yahoo.com>
parents:
44140
diff
changeset
|
511 |
44084
84f2becbd106
lfs: rename a variable to clarify its use
Matt Harbison <matt_harbison@yahoo.com>
parents:
44077
diff
changeset
|
512 with contextlib.closing(self.urlopener.open(request)) as res: |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
513 contentlength = res.info().get(b"content-length") |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
514 ui = self.ui # Shorten debug lines |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
515 if self.ui.debugflag: |
44084
84f2becbd106
lfs: rename a variable to clarify its use
Matt Harbison <matt_harbison@yahoo.com>
parents:
44077
diff
changeset
|
516 ui.debug(b'Status: %d\n' % res.status) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
517 # lfs-test-server and hg serve return headers in different |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
518 # order |
44084
84f2becbd106
lfs: rename a variable to clarify its use
Matt Harbison <matt_harbison@yahoo.com>
parents:
44077
diff
changeset
|
519 headers = pycompat.bytestr(res.info()).strip() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
520 ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
521 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
522 if action == b'download': |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
523 # If downloading blobs, store downloaded data to local |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
524 # blobstore |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
525 localstore.download(oid, res, contentlength) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
526 else: |
44086
ffac09da7a19
lfs: avoid quadratic performance in processing server responses
Matt Harbison <matt_harbison@yahoo.com>
parents:
44085
diff
changeset
|
527 blocks = [] |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
528 while True: |
44084
84f2becbd106
lfs: rename a variable to clarify its use
Matt Harbison <matt_harbison@yahoo.com>
parents:
44077
diff
changeset
|
529 data = res.read(1048576) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
530 if not data: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
531 break |
44086
ffac09da7a19
lfs: avoid quadratic performance in processing server responses
Matt Harbison <matt_harbison@yahoo.com>
parents:
44085
diff
changeset
|
532 blocks.append(data) |
ffac09da7a19
lfs: avoid quadratic performance in processing server responses
Matt Harbison <matt_harbison@yahoo.com>
parents:
44085
diff
changeset
|
533 |
ffac09da7a19
lfs: avoid quadratic performance in processing server responses
Matt Harbison <matt_harbison@yahoo.com>
parents:
44085
diff
changeset
|
534 response = b"".join(blocks) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
535 if response: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
536 ui.debug(b'lfs %s response: %s' % (action, response)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
537 except util.urlerr.httperror as ex: |
35734
b4e1d0654736
lfs: dump the full response on httperror in debug mode
Matt Harbison <matt_harbison@yahoo.com>
parents:
35733
diff
changeset
|
538 if self.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
539 self.ui.debug( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
540 b'%s: %s\n' % (oid, ex.read()) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
541 ) # XXX: also bytes? |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
542 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
543 _(b'LFS HTTP error: %s (oid=%s, action=%s)') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
544 % (stringutil.forcebytestr(ex), oid, action) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
545 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
546 except util.urlerr.urlerror as ex: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
547 hint = _(b'attempted connection to %s') % pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
548 util.urllibcompat.getfullurl(request) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
549 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
550 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
551 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
552 ) |
44139
2ad4e8aefcf4
lfs: explicitly close the file handle for the blob being uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
44138
diff
changeset
|
553 finally: |
2ad4e8aefcf4
lfs: explicitly close the file handle for the blob being uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
44138
diff
changeset
|
554 if request.data: |
2ad4e8aefcf4
lfs: explicitly close the file handle for the blob being uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
44138
diff
changeset
|
555 request.data.close() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
556 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
557 def _batch(self, pointers, localstore, action): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
558 if action not in [b'upload', b'download']: |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
559 raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
560 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
561 response = self._batchrequest(pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
562 objects = self._extractobjects(response, pointers, action) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
563 total = sum(x.get(b'size', 0) for x in objects) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
564 sizes = {} |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
565 for obj in objects: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
566 sizes[obj.get(b'oid')] = obj.get(b'size', 0) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
567 topic = { |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
568 b'upload': _(b'lfs uploading'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
569 b'download': _(b'lfs downloading'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
570 }[action] |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
571 if len(objects) > 1: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
572 self.ui.note( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
573 _(b'lfs: need to transfer %d objects (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
574 % (len(objects), util.bytecount(total)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
575 ) |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
576 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
577 def transfer(chunk): |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
578 for obj in chunk: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
579 objsize = obj.get(b'size', 0) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
580 if self.ui.verbose: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
581 if action == b'download': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
582 msg = _(b'lfs: downloading %s (%s)\n') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
583 elif action == b'upload': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
584 msg = _(b'lfs: uploading %s (%s)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
585 self.ui.note( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
586 msg % (obj.get(b'oid'), util.bytecount(objsize)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
587 ) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
588 retry = self.retry |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
589 while True: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
590 try: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
591 self._basictransfer(obj, action, localstore) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
592 yield 1, obj.get(b'oid') |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
593 break |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
594 except socket.error as ex: |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
595 if retry > 0: |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
596 self.ui.note( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
597 _(b'lfs: failed: %r (remaining retry %d)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
598 % (stringutil.forcebytestr(ex), retry) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
599 ) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
600 retry -= 1 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
601 continue |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
602 raise |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
603 |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49544
diff
changeset
|
604 # Until https multiplexing gets sorted out. It's not clear if |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49544
diff
changeset
|
605 # ConnectionManager.set_ready() is externally synchronized for thread |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49544
diff
changeset
|
606 # safety with Windows workers. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
607 if self.ui.configbool(b'experimental', b'lfs.worker-enable'): |
49544
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
608 # The POSIX workers are forks of this process, so before spinning |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
609 # them up, close all pooled connections. Otherwise, there's no way |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
610 # to coordinate between them about who is using what, and the |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
611 # transfers will get corrupted. |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
612 # |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
613 # TODO: add a function to keepalive.ConnectionManager to mark all |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
614 # ready connections as in use, and roll that back after the fork? |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
615 # That would allow the existing pool of connections in this process |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
616 # to be preserved. |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49544
diff
changeset
|
617 def prefork(): |
49544
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
618 for h in self.urlopener.handlers: |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
619 getattr(h, "close_all", lambda: None)() |
abf471862b8e
lfs: fix blob corruption when tranferring with workers on posix
Matt Harbison <matt_harbison@yahoo.com>
parents:
49526
diff
changeset
|
620 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
621 oids = worker.worker( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
622 self.ui, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
623 0.1, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
624 transfer, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
625 (), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
626 sorted(objects, key=lambda o: o.get(b'oid')), |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49544
diff
changeset
|
627 prefork=prefork, |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
628 ) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
629 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
630 oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
631 |
44077
05881d002cb2
lfs: add "bytes" as the unit to the upload/download progress bar
Matt Harbison <matt_harbison@yahoo.com>
parents:
43506
diff
changeset
|
632 with self.ui.makeprogress( |
05881d002cb2
lfs: add "bytes" as the unit to the upload/download progress bar
Matt Harbison <matt_harbison@yahoo.com>
parents:
43506
diff
changeset
|
633 topic, unit=_(b"bytes"), total=total |
05881d002cb2
lfs: add "bytes" as the unit to the upload/download progress bar
Matt Harbison <matt_harbison@yahoo.com>
parents:
43506
diff
changeset
|
634 ) as progress: |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
635 progress.update(0) |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
636 processed = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
637 blobs = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
638 for _one, oid in oids: |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
639 processed += sizes[oid] |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
640 blobs += 1 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
641 progress.update(processed) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
642 self.ui.note(_(b'lfs: processed: %s\n') % oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
643 |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
644 if blobs > 0: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
645 if action == b'upload': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
646 self.ui.status( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
647 _(b'lfs: uploaded %d files (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
648 % (blobs, util.bytecount(processed)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
649 ) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
650 elif action == b'download': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
651 self.ui.status( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
652 _(b'lfs: downloaded %d files (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
653 % (blobs, util.bytecount(processed)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
654 ) |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
655 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
656 def __del__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
657 # copied from mercurial/httppeer.py |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
658 urlopener = getattr(self, 'urlopener', None) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
659 if urlopener: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
660 for h in urlopener.handlers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
661 h.close() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
662 getattr(h, "close_all", lambda: None)() |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
663 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
664 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48934
diff
changeset
|
665 class _dummyremote: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
666 """Dummy store storing blobs to temp directory.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
667 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
668 def __init__(self, repo, url): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
669 fullpath = repo.vfs.join(b'lfs', url.path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
670 self.vfs = lfsvfs(fullpath) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
671 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
672 def writebatch(self, pointers, fromstore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
673 for p in _deduplicate(pointers): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
674 content = fromstore.read(p.oid(), verify=True) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
675 with self.vfs(p.oid(), b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
676 fp.write(content) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
677 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
678 def readbatch(self, pointers, tostore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
679 for p in _deduplicate(pointers): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
680 with self.vfs(p.oid(), b'rb') as fp: |
44085
0ee0a3f6a990
lfs: check content length after downloading content
Matt Harbison <matt_harbison@yahoo.com>
parents:
44084
diff
changeset
|
681 tostore.download(p.oid(), fp, None) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
682 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
683 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48934
diff
changeset
|
684 class _nullremote: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
685 """Null store storing blobs to /dev/null.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
686 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
687 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
688 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
689 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
690 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
691 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
692 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
693 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
694 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
695 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
696 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48934
diff
changeset
|
697 class _promptremote: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
698 """Prompt user to set lfs.url when accessed.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
699 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
700 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
701 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
702 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
703 def writebatch(self, pointers, fromstore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
704 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
705 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
706 def readbatch(self, pointers, tostore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
707 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
708 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
709 def _prompt(self): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
710 raise error.Abort(_(b'lfs.url needs to be configured')) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
711 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
712 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
713 _storemap = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
714 b'https': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
715 b'http': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
716 b'file': _dummyremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
717 b'null': _nullremote, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
718 None: _promptremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
719 } |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
720 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
721 |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
722 def _deduplicate(pointers): |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
723 """Remove any duplicate oids that exist in the list""" |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
724 reduced = util.sortdict() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
725 for p in pointers: |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
726 reduced[p.oid()] = p |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
727 return reduced.values() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
728 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
729 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
730 def _verify(oid, content): |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
731 realoid = hex(hashlib.sha256(content).digest()) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
732 if realoid != oid: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
733 raise LfsCorruptionError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
734 _(b'detected corrupt lfs object: %s') % oid, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
735 hint=_(b'run hg verify'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
736 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
737 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
738 |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
739 def remote(repo, remote=None): |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
740 """remotestore factory. return a store in _storemap depending on config |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
741 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
742 If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
743 infer the endpoint, based on the remote repository using the same path |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
744 adjustments as git. As an extension, 'http' is supported as well so that |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
745 ``hg serve`` works out of the box. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
746 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
747 https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
748 """ |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
749 lfsurl = repo.ui.config(b'lfs', b'url') |
46907
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
750 url = urlutil.url(lfsurl or b'') |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
751 if lfsurl is None: |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
752 if remote: |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
753 path = remote |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
754 elif util.safehasattr(repo, b'_subtoppath'): |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
755 # The pull command sets this during the optional update phase, which |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
756 # tells exactly where the pull originated, whether 'paths.default' |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
757 # or explicit. |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
758 path = repo._subtoppath |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
759 else: |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
760 # TODO: investigate 'paths.remote:lfsurl' style path customization, |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
761 # and fall back to inferring from 'paths.remote' if unspecified. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
762 path = repo.ui.config(b'paths', b'default') or b'' |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
763 |
46907
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
764 defaulturl = urlutil.url(path) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
765 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
766 # TODO: support local paths as well. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
767 # TODO: consider the ssh -> https transformation that git applies |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
768 if defaulturl.scheme in (b'http', b'https'): |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
769 if defaulturl.path and defaulturl.path[:-1] != b'/': |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
770 defaulturl.path += b'/' |
37691
d241e6632669
lfs: fix the inferred remote store path when using a --prefix
Matt Harbison <matt_harbison@yahoo.com>
parents:
37565
diff
changeset
|
771 defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs' |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
772 |
46907
ffd3e823a7e5
urlutil: extract `url` related code from `util` into the new module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46113
diff
changeset
|
773 url = urlutil.url(bytes(defaulturl)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
774 repo.ui.note(_(b'lfs: assuming remote store: %s\n') % url) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
775 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
776 scheme = url.scheme |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
777 if scheme not in _storemap: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
778 raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
779 return _storemap[scheme](repo, url) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
780 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
781 |
39777
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39457
diff
changeset
|
782 class LfsRemoteError(error.StorageError): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
783 pass |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
784 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
785 |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
786 class LfsCorruptionError(error.Abort): |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
787 """Raised when a corrupt blob is detected, aborting an operation |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
788 |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
789 It exists to allow specialized handling on the server side.""" |